+--------------------------------------------------------------------------------+
+                      Kubernetes Configuration @1681216842                      +
+--------------------------------------------------------------------------------+
{
  "id": 1,
  "pool": "RaidZ1",
  "cluster_cidr": "172.16.0.0/16",
  "service_cidr": "172.17.0.0/16",
  "cluster_dns_ip": "172.17.0.10",
  "route_v4_interface": null,
  "route_v4_gateway": null,
  "route_v6_interface": null,
  "route_v6_gateway": null,
  "node_ip": "0.0.0.0",
  "configure_gpus": true,
  "servicelb": true,
  "validate_host_path": true,
  "passthrough_mode": false,
  "dataset": "RaidZ1/ix-applications"
}
debug finished in 0 seconds for Kubernetes Configuration


+--------------------------------------------------------------------------------+
+                     k3s kubectl describe nodes @1681216842                     +
+--------------------------------------------------------------------------------+
Name:               ix-truenas
Roles:              control-plane,master
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/os=linux
                    egress.k3s.io/cluster=true
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=ix-truenas
                    kubernetes.io/os=linux
                    node-role.kubernetes.io/control-plane=true
                    node-role.kubernetes.io/master=true
                    openebs.io/nodeid=ix-truenas
                    openebs.io/nodename=ix-truenas
Annotations:        csi.volume.kubernetes.io/nodeid: {"zfs.csi.openebs.io":"ix-truenas"}
                    k3s.io/node-args:
                      ["server","--cluster-cidr","172.16.0.0/16","--cluster-dns","172.17.0.10","--data-dir","/mnt/RaidZ1/ix-applications/k3s","--kube-apiserver-...
                    k3s.io/node-config-hash: AIGRDHHQBW2TXIIYCCB6SBC3YNORJLIE3EPHG44QTLR35VZOXLIQ====
                    k3s.io/node-env: {"K3S_DATA_DIR":"/mnt/RaidZ1/ix-applications/k3s/data/11f65b70714b22f8cffe9fe6db17b08dba2dc8b47550f6dc79f62f1aad3afb18"}
                    node.alpha.kubernetes.io/ttl: 0
                    volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp:  Mon, 09 Jan 2023 20:54:04 +0100
Taints:             <none>
Unschedulable:      false
Lease:
  HolderIdentity:  ix-truenas
  AcquireTime:     <unset>
  RenewTime:       Tue, 11 Apr 2023 14:40:35 +0200
Conditions:
  Type             Status  LastHeartbeatTime                 LastTransitionTime                Reason                       Message
  ----             ------  -----------------                 ------------------                ------                       -------
  MemoryPressure   False   Tue, 11 Apr 2023 14:38:23 +0200   Mon, 09 Jan 2023 20:54:04 +0100   KubeletHasSufficientMemory   kubelet has sufficient memory available
  DiskPressure     False   Tue, 11 Apr 2023 14:38:23 +0200   Mon, 09 Jan 2023 20:54:04 +0100   KubeletHasNoDiskPressure     kubelet has no disk pressure
  PIDPressure      False   Tue, 11 Apr 2023 14:38:23 +0200   Mon, 09 Jan 2023 20:54:04 +0100   KubeletHasSufficientPID      kubelet has sufficient PID available
  Ready            True    Tue, 11 Apr 2023 14:38:23 +0200   Tue, 11 Apr 2023 14:38:13 +0200   KubeletReady                 kubelet is posting ready status. AppArmor enabled
Addresses:
  InternalIP:  192.168.2.105
  Hostname:    ix-truenas
Capacity:
  cpu:                 4
  ephemeral-storage:   18171799168Ki
  gpu.intel.com/i915:  5
  hugepages-1Gi:       0
  hugepages-2Mi:       0
  memory:              15901792Ki
  pods:                250
Allocatable:
  cpu:                 4
  ephemeral-storage:   17677526216767
  gpu.intel.com/i915:  5
  hugepages-1Gi:       0
  hugepages-2Mi:       0
  memory:              15901792Ki
  pods:                250
System Info:
  Machine ID:                 2de876b8cf1c4fd6ac85bff2384e3ada
  System UUID:                00000000-0000-0000-0000-d05099fbf079
  Boot ID:                    290f3407-5042-4c61-8207-5b1c6953b733
  Kernel Version:             5.15.79+truenas
  OS Image:                   Debian GNU/Linux 11 (bullseye)
  Operating System:           linux
  Architecture:               amd64
  Container Runtime Version:  docker://Unknown
  Kubelet Version:            v1.25.3+k3s-9afcd6b9-dirty
  Kube-Proxy Version:         v1.25.3+k3s-9afcd6b9-dirty
PodCIDR:                      172.16.0.0/16
PodCIDRs:                     172.16.0.0/16
Non-terminated Pods:          (8 in total)
  Namespace                   Name                                    CPU Requests  CPU Limits  Memory Requests  Memory Limits  Age
  ---------                   ----                                    ------------  ----------  ---------------  -------------  ---
  kube-system                 openebs-zfs-node-bwxzr                  0 (0%)        0 (0%)      0 (0%)           0 (0%)         39m
  metallb-system              speaker-rcq54                           0 (0%)        0 (0%)      0 (0%)           0 (0%)         39m
  kube-system                 openebs-zfs-controller-0                0 (0%)        0 (0%)      0 (0%)           0 (0%)         39m
  kube-system                 intel-gpu-plugin-zvqhc                  0 (0%)        0 (0%)      0 (0%)           0 (0%)         39m
  kube-system                 coredns-75fc8f8fff-dnkbk                100m (2%)     0 (0%)      70Mi (0%)        170Mi (1%)     39m
  prometheus-operator         prometheus-operator-5dcffb7cb8-q6xxq    100m (2%)     200m (5%)   100Mi (0%)       200Mi (1%)     39m
  metallb-system              controller-84d6d4db45-2jc2g             0 (0%)        0 (0%)      0 (0%)           0 (0%)         39m
  ix-netdata                  netdata-7bc8fbb7b4-8vz7s                0 (0%)        0 (0%)      0 (0%)           0 (0%)         39m
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource            Requests    Limits
  --------            --------    ------
  cpu                 200m (5%)   200m (5%)
  memory              170Mi (1%)  370Mi (2%)
  ephemeral-storage   0 (0%)      0 (0%)
  hugepages-1Gi       0 (0%)      0 (0%)
  hugepages-2Mi       0 (0%)      0 (0%)
  gpu.intel.com/i915  0           0
Events:
  Type     Reason                   Age                From             Message
  ----     ------                   ----               ----             -------
  Normal   NodeNotReady             80m                kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeHasNoDiskPressure    80m                kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Warning  Rebooted                 80m                kubelet          Node ix-truenas has been rebooted, boot id: 7ed891ef-2297-44ad-814a-39c6bd4e2aec
  Normal   NodeHasSufficientPID     80m                kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Normal   NodeHasSufficientMemory  80m                kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   Starting                 80m                kubelet          Starting kubelet.
  Normal   NodeAllocatableEnforced  80m                kubelet          Updated Node Allocatable limit across pods
  Normal   NodeReady                80m                kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           80m                node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
  Normal   Starting                 69m                kubelet          Starting kubelet.
  Normal   NodeAllocatableEnforced  69m                kubelet          Updated Node Allocatable limit across pods
  Normal   NodeHasSufficientMemory  69m                kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   NodeHasNoDiskPressure    69m                kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Normal   NodeHasSufficientPID     69m                kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Warning  Rebooted                 69m                kubelet          Node ix-truenas has been rebooted, boot id: 24de3026-11d5-4f09-abbe-5590ce1b96eb
  Normal   NodeNotReady             69m                kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeReady                69m                kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           69m                node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
  Normal   Starting                 51m                kubelet          Starting kubelet.
  Normal   NodeHasSufficientMemory  51m                kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   NodeHasNoDiskPressure    51m                kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Normal   NodeHasSufficientPID     51m                kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Normal   NodeAllocatableEnforced  51m                kubelet          Updated Node Allocatable limit across pods
  Warning  Rebooted                 51m (x2 over 51m)  kubelet          Node ix-truenas has been rebooted, boot id: 66572a2a-21a2-4c9f-90e5-6e694bc5b5cc
  Normal   NodeNotReady             51m (x2 over 51m)  kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeReady                50m                kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           50m                node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
  Normal   NodeHasSufficientPID     39m                kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Normal   NodeHasSufficientMemory  39m                kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   NodeHasNoDiskPressure    39m                kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Normal   Starting                 39m                kubelet          Starting kubelet.
  Normal   NodeAllocatableEnforced  39m                kubelet          Updated Node Allocatable limit across pods
  Warning  Rebooted                 39m                kubelet          Node ix-truenas has been rebooted, boot id: 948b762e-d99c-4e2d-988e-88bb8a8dc78e
  Normal   NodeNotReady             39m                kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeReady                39m                kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           39m                node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
  Normal   Starting                 33m                kubelet          Starting kubelet.
  Normal   NodeHasSufficientMemory  33m                kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   NodeHasNoDiskPressure    33m                kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Normal   NodeHasSufficientPID     33m                kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Normal   NodeAllocatableEnforced  33m                kubelet          Updated Node Allocatable limit across pods
  Warning  Rebooted                 33m                kubelet          Node ix-truenas has been rebooted, boot id: 310a5bc3-9c9e-47d9-8624-3bb8e3574a33
  Normal   NodeNotReady             33m                kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeReady                33m                kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           33m                node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
  Normal   Starting                 30m                kubelet          Starting kubelet.
  Normal   NodeHasSufficientMemory  30m                kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   NodeHasNoDiskPressure    30m                kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Normal   NodeHasSufficientPID     30m                kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Warning  Rebooted                 30m                kubelet          Node ix-truenas has been rebooted, boot id: c2f77b86-6e2d-4aee-9508-35481d2259cf
  Normal   NodeAllocatableEnforced  30m                kubelet          Updated Node Allocatable limit across pods
  Normal   NodeNotReady             30m                kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeReady                30m                kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           30m                node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
  Normal   Starting                 2m40s              kubelet          Starting kubelet.
  Normal   NodeHasSufficientMemory  2m40s              kubelet          Node ix-truenas status is now: NodeHasSufficientMemory
  Normal   NodeHasNoDiskPressure    2m40s              kubelet          Node ix-truenas status is now: NodeHasNoDiskPressure
  Normal   NodeHasSufficientPID     2m40s              kubelet          Node ix-truenas status is now: NodeHasSufficientPID
  Normal   NodeAllocatableEnforced  2m40s              kubelet          Updated Node Allocatable limit across pods
  Warning  Rebooted                 2m40s              kubelet          Node ix-truenas has been rebooted, boot id: 290f3407-5042-4c61-8207-5b1c6953b733
  Normal   NodeNotReady             2m39s              kubelet          Node ix-truenas status is now: NodeNotReady
  Normal   NodeReady                2m29s              kubelet          Node ix-truenas status is now: NodeReady
  Normal   RegisteredNode           2m28s              node-controller  Node ix-truenas event: Registered Node ix-truenas in Controller
debug finished in 0 seconds for k3s kubectl describe nodes


+--------------------------------------------------------------------------------+
+k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide @1681216842+
+--------------------------------------------------------------------------------+
NAMESPACE             NAME                                       READY   STATUS            RESTARTS       AGE   IP              NODE         NOMINATED NODE   READINESS GATES
ix-netdata            pod/netdata-7bc8fbb7b4-hcnbb               0/1     TaintToleration   0              69m   <none>          ix-truenas   <none>           <none>
prometheus-operator   pod/prometheus-operator-5dcffb7cb8-xp8th   0/1     TaintToleration   0              69m   <none>          ix-truenas   <none>           <none>
kube-system           pod/openebs-zfs-node-bwxzr                 2/2     Running           6 (25m ago)    39m   192.168.2.105   ix-truenas   <none>           <none>
metallb-system        pod/speaker-rcq54                          1/1     Running           3 (26m ago)    39m   192.168.2.105   ix-truenas   <none>           <none>
kube-system           pod/openebs-zfs-controller-0               5/5     Running           15 (25m ago)   39m   172.16.0.58     ix-truenas   <none>           <none>
kube-system           pod/intel-gpu-plugin-zvqhc                 1/1     Running           3 (26m ago)    39m   172.16.0.57     ix-truenas   <none>           <none>
kube-system           pod/coredns-75fc8f8fff-dnkbk               1/1     Running           3 (26m ago)    39m   172.16.0.60     ix-truenas   <none>           <none>
prometheus-operator   pod/prometheus-operator-5dcffb7cb8-q6xxq   1/1     Running           3 (26m ago)    39m   172.16.0.61     ix-truenas   <none>           <none>
metallb-system        pod/controller-84d6d4db45-2jc2g            1/1     Running           3 (26m ago)    39m   172.16.0.59     ix-truenas   <none>           <none>
ix-netdata            pod/netdata-7bc8fbb7b4-8vz7s               1/1     Running           3 (25m ago)    39m   172.16.0.62     ix-truenas   <none>           <none>

NAMESPACE             NAME                          TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                        AGE   SELECTOR
default               service/kubernetes            ClusterIP   172.17.0.1       <none>        443/TCP                        91d   <none>
kube-system           service/kube-dns              ClusterIP   172.17.0.10      <none>        53/UDP,53/TCP,9153/TCP         91d   k8s-app=kube-dns
metallb-system        service/webhook-service       ClusterIP   172.17.209.192   <none>        443/TCP                        91d   component=controller
prometheus-operator   service/prometheus-operator   ClusterIP   None             <none>        8080/TCP                       91d   app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
kube-system           service/kubelet               ClusterIP   None             <none>        10250/TCP,10255/TCP,4194/TCP   91d   <none>
ix-netdata            service/netdata               NodePort    172.17.55.206    <none>        20489:20489/TCP                2d    app.kubernetes.io/instance=netdata,app.kubernetes.io/name=netdata

NAMESPACE        NAME                              DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR              AGE   CONTAINERS                                     IMAGES                                                                             SELECTOR
kube-system      daemonset.apps/openebs-zfs-node   1         1         1       1            1           <none>                     91d   csi-node-driver-registrar,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0,openebs/zfs-driver:2.0.0   app=openebs-zfs-node
metallb-system   daemonset.apps/speaker            1         1         1       1            1           kubernetes.io/os=linux     91d   speaker                                        quay.io/metallb/speaker:v0.13.7                                                    app=metallb,component=speaker
kube-system      daemonset.apps/intel-gpu-plugin   1         1         1       1            1           kubernetes.io/arch=amd64   91d   intel-gpu-plugin                               intel/intel-gpu-plugin:0.19.0                                                      app=intel-gpu-plugin

NAMESPACE             NAME                                  READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS            IMAGES                                                    SELECTOR
kube-system           deployment.apps/coredns               1/1     1            1           91d   coredns               rancher/mirrored-coredns-coredns:1.9.1                    k8s-app=kube-dns
prometheus-operator   deployment.apps/prometheus-operator   1/1     1            1           91d   prometheus-operator   quay.io/prometheus-operator/prometheus-operator:v0.60.1   app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
metallb-system        deployment.apps/controller            1/1     1            1           91d   controller            quay.io/metallb/controller:v0.13.7                        app=metallb,component=controller
ix-netdata            deployment.apps/netdata               1/1     1            1           2d    netdata               netdata/netdata:v1.38.1                                   app.kubernetes.io/instance=netdata,app.kubernetes.io/name=netdata

NAMESPACE     NAME                                      READY   AGE   CONTAINERS                                                                           IMAGES
kube-system   statefulset.apps/openebs-zfs-controller   1/1     91d   csi-resizer,csi-snapshotter,snapshot-controller,csi-provisioner,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-resizer:v1.2.0,k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0,k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0,k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0,openebs/zfs-driver:2.0.0

NAMESPACE   NAME                                                          PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
            storageclass.storage.k8s.io/ix-storage-class-netdata          zfs.csi.openebs.io   Retain          Immediate           true                   2d
            storageclass.storage.k8s.io/openebs-zfspv-default (default)   zfs.csi.openebs.io   Delete          Immediate           true                   2m36s

NAMESPACE   NAME                            STATUS   AGE
            namespace/kube-system           Active   91d
            namespace/default               Active   91d
            namespace/kube-public           Active   91d
            namespace/kube-node-lease       Active   91d
            namespace/openebs               Active   91d
            namespace/cnpg-system           Active   91d
            namespace/metallb-system        Active   91d
            namespace/prometheus-operator   Active   91d
            namespace/tc-system             Active   91d
            namespace/ix-netdata            Active   2d
debug finished in 1 seconds for k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide


+--------------------------------------------------------------------------------+
+         k3s kubectl describe deployments --all-namespaces @1681216843          +
+--------------------------------------------------------------------------------+
Name:                   coredns
Namespace:              kube-system
CreationTimestamp:      Mon, 09 Jan 2023 20:54:06 +0100
Labels:                 k8s-app=kube-dns
                        kubernetes.io/name=CoreDNS
                        objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:            deployment.kubernetes.io/revision: 1
                        objectset.rio.cattle.io/applied:
                          H4sIAAAAAAAA/6xVUW/bNhD+K8M9S7GVtI0rYA+dna1FV8+ok74URkFTJ4szxePIkxMj0H8fTrIdu02TdtiTZfK743cfvzveg/LmE4ZoyEEOyvs42GSQwNq4AnKYoLe0rdExJFAjq0...
                        objectset.rio.cattle.io/id: 
                        objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                        objectset.rio.cattle.io/owner-name: coredns
                        objectset.rio.cattle.io/owner-namespace: kube-system
Selector:               k8s-app=kube-dns
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  1 max unavailable, 25% max surge
Pod Template:
  Labels:           k8s-app=kube-dns
  Service Account:  coredns
  Containers:
   coredns:
    Image:       rancher/mirrored-coredns-coredns:1.9.1
    Ports:       53/UDP, 53/TCP, 9153/TCP
    Host Ports:  0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
  Volumes:
   config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
   custom-config-volume:
    Type:                       ConfigMap (a volume populated by a ConfigMap)
    Name:                       coredns-custom
    Optional:                   true
  Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
  Priority Class Name:          system-cluster-critical
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   coredns-75fc8f8fff (1/1 replicas created)
Events:          <none>


Name:                   prometheus-operator
Namespace:              prometheus-operator
CreationTimestamp:      Tue, 10 Jan 2023 09:39:20 +0100
Labels:                 app.kubernetes.io/component=controller
                        app.kubernetes.io/name=prometheus-operator
                        app.kubernetes.io/version=0.60.1
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app.kubernetes.io/component=controller
                    app.kubernetes.io/name=prometheus-operator
                    app.kubernetes.io/version=0.60.1
  Annotations:      kubectl.kubernetes.io/default-container: prometheus-operator
  Service Account:  prometheus-operator
  Containers:
   prometheus-operator:
    Image:      quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Port:       8080/TCP
    Host Port:  0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:       <none>
  Volumes:        <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   prometheus-operator-5dcffb7cb8 (1/1 replicas created)
Events:          <none>


Name:                   controller
Namespace:              metallb-system
CreationTimestamp:      Tue, 10 Jan 2023 09:39:20 +0100
Labels:                 app=metallb
                        component=controller
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app=metallb,component=controller
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app=metallb
                    component=controller
  Annotations:      prometheus.io/port: 7472
                    prometheus.io/scrape: true
  Service Account:  controller
  Containers:
   controller:
    Image:       quay.io/metallb/controller:v0.13.7
    Ports:       7472/TCP, 9443/TCP
    Host Ports:  0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    Liveness:   http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
  Volumes:
   cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   controller-84d6d4db45 (1/1 replicas created)
Events:          <none>


Name:               netdata
Namespace:          ix-netdata
CreationTimestamp:  Sun, 09 Apr 2023 14:25:52 +0200
Labels:             app=netdata
                    app.kubernetes.io/managed-by=Helm
                    chart=netdata-1.0.16
                    heritage=Helm
                    release=netdata
                    role=parent
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: netdata
                    meta.helm.sh/release-namespace: ix-netdata
Selector:           app.kubernetes.io/instance=netdata,app.kubernetes.io/name=netdata
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=netdata
                    app.kubernetes.io/name=netdata
  Annotations:      rollme: 5Tu4r
  Service Account:  netdata
  Containers:
   netdata:
    Image:        netdata/netdata:v1.38.1
    Port:         19999/TCP
    Host Port:    0/TCP
    Liveness:     http-get http://:http/api/v1/info delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:    http-get http://:http/api/v1/info delay=10s timeout=5s period=10s #success=2 #failure=5
    Startup:      http-get http://:http/api/v1/info delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:  <none>
    Mounts:
      /etc/netdata from netdataconfig (rw)
      /host/etc/group from group (ro)
      /host/etc/os-release from os-release (ro)
      /host/etc/passwd from user (ro)
      /host/proc from proc (ro)
      /host/sys from sys (ro)
      /var/cache/netdata from netdatacache (rw)
      /var/lib/netdata from netdatalib (rw)
  Volumes:
   netdatacache:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-cache
    HostPathType:  
   netdataconfig:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-config
    HostPathType:  
   netdatalib:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-lib
    HostPathType:  
   os-release:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/os-release
    HostPathType:  
   proc:
    Type:          HostPath (bare host directory volume)
    Path:          /proc
    HostPathType:  
   sys:
    Type:          HostPath (bare host directory volume)
    Path:          /sys
    HostPathType:  
   user:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/passwd
    HostPathType:  
   group:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/group
    HostPathType:  
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   netdata-7bc8fbb7b4 (1/1 replicas created)
Events:          <none>
debug finished in 0 seconds for k3s kubectl describe deployments --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe pods --all-namespaces @1681216843             +
+--------------------------------------------------------------------------------+
Name:             netdata-7bc8fbb7b4-hcnbb
Namespace:        ix-netdata
Priority:         0
Service Account:  netdata
Node:             ix-truenas/
Start Time:       Tue, 11 Apr 2023 13:31:25 +0200
Labels:           app.kubernetes.io/instance=netdata
                  app.kubernetes.io/name=netdata
                  pod-template-hash=7bc8fbb7b4
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.0.35"
                        ],
                        "mac": "a6:d8:52:33:4f:3d",
                        "default": true,
                        "dns": {}
                    }]
                  rollme: 5Tu4r
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/netdata-7bc8fbb7b4
Containers:
  netdata:
    Image:        netdata/netdata:v1.38.1
    Port:         19999/TCP
    Host Port:    0/TCP
    Liveness:     http-get http://:http/api/v1/info delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:    http-get http://:http/api/v1/info delay=10s timeout=5s period=10s #success=2 #failure=5
    Startup:      http-get http://:http/api/v1/info delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:  <none>
    Mounts:
      /etc/netdata from netdataconfig (rw)
      /host/etc/group from group (ro)
      /host/etc/os-release from os-release (ro)
      /host/etc/passwd from user (ro)
      /host/proc from proc (ro)
      /host/sys from sys (ro)
      /var/cache/netdata from netdatacache (rw)
      /var/lib/netdata from netdatalib (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-cnzw7 (ro)
Volumes:
  netdatacache:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-cache
    HostPathType:  
  netdataconfig:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-config
    HostPathType:  
  netdatalib:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-lib
    HostPathType:  
  os-release:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/os-release
    HostPathType:  
  proc:
    Type:          HostPath (bare host directory volume)
    Path:          /proc
    HostPathType:  
  sys:
    Type:          HostPath (bare host directory volume)
    Path:          /sys
    HostPathType:  
  user:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/passwd
    HostPathType:  
  group:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/group
    HostPathType:  
  kube-api-access-cnzw7:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason           Age                From               Message
  ----     ------           ----               ----               -------
  Normal   Scheduled        69m                default-scheduler  Successfully assigned ix-netdata/netdata-7bc8fbb7b4-hcnbb to ix-truenas
  Normal   AddedInterface   69m                multus             Add eth0 [172.16.0.29/16] from ix-net
  Normal   Pulled           69m                kubelet            Container image "netdata/netdata:v1.38.1" already present on machine
  Normal   Created          69m                kubelet            Created container netdata
  Normal   Started          69m                kubelet            Started container netdata
  Warning  NetworkNotReady  51m (x5 over 51m)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      50m (x5 over 51m)  kubelet            MountVolume.SetUp failed for volume "kube-api-access-cnzw7" : object "ix-netdata"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   50m                kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   50m                multus             Add eth0 [172.16.0.35/16] from ix-net
  Normal   Pulled           50m                kubelet            Container image "netdata/netdata:v1.38.1" already present on machine
  Normal   Created          50m                kubelet            Created container netdata
  Normal   Started          50m                kubelet            Started container netdata
  Warning  TaintToleration  39m                kubelet            Predicate TaintToleration failed
  Warning  FailedMount      39m                kubelet            MountVolume.SetUp failed for volume "kube-api-access-cnzw7" : object "ix-netdata"/"kube-root-ca.crt" not registered


Name:             prometheus-operator-5dcffb7cb8-xp8th
Namespace:        prometheus-operator
Priority:         0
Service Account:  prometheus-operator
Node:             ix-truenas/
Start Time:       Tue, 11 Apr 2023 13:31:25 +0200
Labels:           app.kubernetes.io/component=controller
                  app.kubernetes.io/name=prometheus-operator
                  app.kubernetes.io/version=0.60.1
                  pod-template-hash=5dcffb7cb8
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.0.38"
                        ],
                        "mac": "96:a5:41:0f:7d:94",
                        "default": true,
                        "dns": {}
                    }]
                  kubectl.kubernetes.io/default-container: prometheus-operator
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/prometheus-operator-5dcffb7cb8
Containers:
  prometheus-operator:
    Image:      quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Port:       8080/TCP
    Host Port:  0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-wkn58 (ro)
Volumes:
  kube-api-access-wkn58:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason           Age                From               Message
  ----     ------           ----               ----               -------
  Normal   Scheduled        69m                default-scheduler  Successfully assigned prometheus-operator/prometheus-operator-5dcffb7cb8-xp8th to ix-truenas
  Normal   AddedInterface   69m                multus             Add eth0 [172.16.0.28/16] from ix-net
  Normal   Pulled           69m                kubelet            Container image "quay.io/prometheus-operator/prometheus-operator:v0.60.1" already present on machine
  Normal   Created          69m                kubelet            Created container prometheus-operator
  Normal   Started          69m                kubelet            Started container prometheus-operator
  Warning  NetworkNotReady  51m (x5 over 51m)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      50m (x5 over 51m)  kubelet            MountVolume.SetUp failed for volume "kube-api-access-wkn58" : object "prometheus-operator"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   50m                kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   50m                multus             Add eth0 [172.16.0.38/16] from ix-net
  Normal   Pulled           50m                kubelet            Container image "quay.io/prometheus-operator/prometheus-operator:v0.60.1" already present on machine
  Normal   Created          50m                kubelet            Created container prometheus-operator
  Normal   Started          50m                kubelet            Started container prometheus-operator
  Warning  TaintToleration  39m                kubelet            Predicate TaintToleration failed
  Warning  FailedMount      39m (x2 over 39m)  kubelet            MountVolume.SetUp failed for volume "kube-api-access-wkn58" : object "prometheus-operator"/"kube-root-ca.crt" not registered


Name:                 openebs-zfs-node-bwxzr
Namespace:            kube-system
Priority:             900001000
Priority Class Name:  openebs-zfs-csi-node-critical
Service Account:      openebs-zfs-node-sa
Node:                 ix-truenas/192.168.2.105
Start Time:           Tue, 11 Apr 2023 14:01:41 +0200
Labels:               app=openebs-zfs-node
                      controller-revision-hash=66bcbb6975
                      openebs.io/component-name=openebs-zfs-node
                      openebs.io/version=ci
                      pod-template-generation=1
                      role=openebs-zfs
Annotations:          <none>
Status:               Running
IP:                   192.168.2.105
IPs:
  IP:           192.168.2.105
Controlled By:  DaemonSet/openebs-zfs-node
Containers:
  csi-node-driver-registrar:
    Container ID:  docker://0b3856f9340cd260d6ff4ef04c285eadf478d6443cf07d52795ff483455b1329
    Image:         k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:f9bcee63734b7b01555ee8fc8fb01ac2922478b2c8934bf8d468dd2916edc405
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:06 +0200
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:10:04 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zqmlw (ro)
  openebs-zfs-plugin:
    Container ID:  docker://fc0253809d9172535d2a3be01eafc5e9e6974c2f2f253c38bd019a4b3b41463e
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:08 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Tue, 11 Apr 2023 14:10:05 +0200
      Finished:     Tue, 11 Apr 2023 14:14:47 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zqmlw (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
  encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
  chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
  host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  Directory
  kube-api-access-zqmlw:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type     Reason          Age                    From               Message
  ----     ------          ----                   ----               -------
  Normal   Scheduled       39m                    default-scheduler  Successfully assigned kube-system/openebs-zfs-node-bwxzr to ix-truenas
  Normal   Pulled          39m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0" already present on machine
  Normal   Created         38m                    kubelet            Created container csi-node-driver-registrar
  Normal   Started         38m                    kubelet            Started container csi-node-driver-registrar
  Normal   Pulled          38m                    kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created         38m                    kubelet            Created container openebs-zfs-plugin
  Normal   Started         38m                    kubelet            Started container openebs-zfs-plugin
  Normal   SandboxChanged  33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   Pulled          33m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0" already present on machine
  Normal   Created         33m                    kubelet            Created container csi-node-driver-registrar
  Normal   Started         33m                    kubelet            Started container csi-node-driver-registrar
  Normal   Pulled          33m                    kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created         33m                    kubelet            Created container openebs-zfs-plugin
  Normal   Started         33m                    kubelet            Started container openebs-zfs-plugin
  Warning  FailedMount     30m                    kubelet            MountVolume.SetUp failed for volume "chroot-zfs" : object "kube-system"/"openebs-zfspv-bin" not registered
  Warning  FailedMount     30m                    kubelet            MountVolume.SetUp failed for volume "kube-api-access-zqmlw" : failed to sync configmap cache: timed out waiting for the condition
  Warning  FailedMount     30m                    kubelet            MountVolume.SetUp failed for volume "chroot-zfs" : failed to sync configmap cache: timed out waiting for the condition
  Normal   SandboxChanged  30m (x2 over 30m)      kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   Pulled          30m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0" already present on machine
  Normal   Created         30m                    kubelet            Created container csi-node-driver-registrar
  Normal   Started         30m                    kubelet            Started container csi-node-driver-registrar
  Normal   Pulled          30m                    kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created         30m                    kubelet            Created container openebs-zfs-plugin
  Normal   Started         30m                    kubelet            Started container openebs-zfs-plugin
  Normal   SandboxChanged  2m39s (x2 over 2m39s)  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   Pulled          2m38s                  kubelet            Container image "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0" already present on machine
  Normal   Created         2m37s                  kubelet            Created container csi-node-driver-registrar
  Normal   Started         2m37s                  kubelet            Started container csi-node-driver-registrar
  Normal   Pulled          2m37s                  kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created         2m36s                  kubelet            Created container openebs-zfs-plugin
  Normal   Started         2m35s                  kubelet            Started container openebs-zfs-plugin


Name:             speaker-rcq54
Namespace:        metallb-system
Priority:         0
Service Account:  speaker
Node:             ix-truenas/192.168.2.105
Start Time:       Tue, 11 Apr 2023 14:01:41 +0200
Labels:           app=metallb
                  component=speaker
                  controller-revision-hash=67d677d8b6
                  pod-template-generation=1
Annotations:      prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Running
IP:               192.168.2.105
IPs:
  IP:           192.168.2.105
Controlled By:  DaemonSet/speaker
Containers:
  speaker:
    Container ID:  docker://feddfb1a978ecf0a82faa0eebbce0a2d4f3ca1f5a8f5a0657f8d3a73e1b6fe78
    Image:         quay.io/metallb/speaker:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/speaker@sha256:aeab5aeeb25c7ca31a1c468a35b1db4513cd6199bf376bdf19ff5689d2ebad28
    Ports:         7472/TCP, 7946/TCP, 7946/UDP
    Host Ports:    7472/TCP, 7946/TCP, 7946/UDP
    Args:
      --port=7472
      --log-level=info
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:06 +0200
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:10:03 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_NODE_NAME:       (v1:spec.nodeName)
      METALLB_HOST:            (v1:status.hostIP)
      METALLB_ML_BIND_ADDR:    (v1:status.podIP)
      METALLB_ML_LABELS:      app=metallb,component=speaker
      METALLB_ML_SECRET_KEY:  <set to the key 'secretkey' in secret 'memberlist'>  Optional: false
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-9tqfl (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-9tqfl:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type    Reason          Age                    From               Message
  ----    ------          ----                   ----               -------
  Normal  Scheduled       39m                    default-scheduler  Successfully assigned metallb-system/speaker-rcq54 to ix-truenas
  Normal  Pulled          39m                    kubelet            Container image "quay.io/metallb/speaker:v0.13.7" already present on machine
  Normal  Created         39m                    kubelet            Created container speaker
  Normal  Started         38m                    kubelet            Started container speaker
  Normal  SandboxChanged  33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal  Pulled          33m                    kubelet            Container image "quay.io/metallb/speaker:v0.13.7" already present on machine
  Normal  Created         33m                    kubelet            Created container speaker
  Normal  Started         33m                    kubelet            Started container speaker
  Normal  SandboxChanged  30m (x2 over 30m)      kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal  Pulled          30m                    kubelet            Container image "quay.io/metallb/speaker:v0.13.7" already present on machine
  Normal  Created         30m                    kubelet            Created container speaker
  Normal  Started         30m                    kubelet            Started container speaker
  Normal  SandboxChanged  2m39s (x2 over 2m39s)  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal  Pulled          2m38s                  kubelet            Container image "quay.io/metallb/speaker:v0.13.7" already present on machine
  Normal  Created         2m38s                  kubelet            Created container speaker
  Normal  Started         2m37s                  kubelet            Started container speaker


Name:                 openebs-zfs-controller-0
Namespace:            kube-system
Priority:             900000000
Priority Class Name:  openebs-zfs-csi-controller-critical
Service Account:      openebs-zfs-controller-sa
Node:                 ix-truenas/192.168.2.105
Start Time:           Tue, 11 Apr 2023 14:01:41 +0200
Labels:               app=openebs-zfs-controller
                      controller-revision-hash=openebs-zfs-controller-698698d48b
                      openebs.io/component-name=openebs-zfs-controller
                      openebs.io/version=ci
                      role=openebs-zfs
                      statefulset.kubernetes.io/pod-name=openebs-zfs-controller-0
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.58"
                            ],
                            "mac": "52:e7:9d:6b:53:f9",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.0.58
IPs:
  IP:           172.16.0.58
Controlled By:  StatefulSet/openebs-zfs-controller
Containers:
  csi-resizer:
    Container ID:  docker://f0a2eafdda3b6628378002b4b0bbdae3ba715d33b96ceebaf98af9e8d206a226
    Image:         k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:16 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Tue, 11 Apr 2023 14:10:11 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-nqtx7 (ro)
  csi-snapshotter:
    Container ID:  docker://7a729ff831b420472ce82ecb46fa4efe53d5b9e62d3be7b72f9b8d98f39022ca
    Image:         k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-snapshotter@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:17 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Tue, 11 Apr 2023 14:10:12 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-nqtx7 (ro)
  snapshot-controller:
    Container ID:  docker://8704603c35d3dd32659e6a4b8c02172d197973594b5a96848607ba0cee72d623
    Image:         k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/snapshot-controller@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --leader-election=true
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:17 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Tue, 11 Apr 2023 14:10:12 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-nqtx7 (ro)
  csi-provisioner:
    Container ID:  docker://6001b6e8dd43a0f2e15938b5f7b3b1b00571c536de2936b1fce0037f62eff51f
    Image:         k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-provisioner@sha256:6477988532358148d2e98f7c747db4e9250bbc7ad2664bf666348abf9ee1f5aa
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:19 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Tue, 11 Apr 2023 14:10:13 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:  kube-system (v1:metadata.namespace)
      POD_NAME:   openebs-zfs-controller-0 (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-nqtx7 (ro)
  openebs-zfs-plugin:
    Container ID:  docker://42f866aca9ed236e1cff9d23c279b32ba1d3032d029f5569b073a57ce47893b1
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:19 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Tue, 11 Apr 2023 14:10:14 +0200
      Finished:     Tue, 11 Apr 2023 14:14:47 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-nqtx7 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-nqtx7:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason           Age                    From               Message
  ----     ------           ----                   ----               -------
  Normal   Scheduled        39m                    default-scheduler  Successfully assigned kube-system/openebs-zfs-controller-0 to ix-truenas
  Normal   AddedInterface   38m                    multus             Add eth0 [172.16.0.44/16] from ix-net
  Normal   Pulled           38m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-resizer:v1.2.0" already present on machine
  Normal   Created          38m                    kubelet            Created container csi-resizer
  Normal   Started          38m                    kubelet            Started container csi-resizer
  Normal   Pulled           38m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" already present on machine
  Normal   Created          38m                    kubelet            Created container csi-snapshotter
  Normal   Pulled           38m                    kubelet            Container image "k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0" already present on machine
  Normal   Started          38m                    kubelet            Started container csi-snapshotter
  Normal   Created          38m                    kubelet            Created container snapshot-controller
  Normal   Started          38m                    kubelet            Started container snapshot-controller
  Normal   Pulled           38m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0" already present on machine
  Normal   Created          38m                    kubelet            Created container csi-provisioner
  Normal   Started          38m                    kubelet            Started container csi-provisioner
  Normal   Pulled           38m                    kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created          38m                    kubelet            Created container openebs-zfs-plugin
  Normal   Started          38m                    kubelet            Started container openebs-zfs-plugin
  Warning  NetworkNotReady  33m (x5 over 33m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   33m                    multus             Add eth0 [172.16.0.45/16] from ix-net
  Normal   Pulled           33m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-resizer:v1.2.0" already present on machine
  Normal   Created          33m                    kubelet            Created container csi-resizer
  Normal   Started          33m                    kubelet            Started container csi-resizer
  Normal   Pulled           33m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" already present on machine
  Normal   Created          33m                    kubelet            Created container csi-snapshotter
  Normal   Started          33m                    kubelet            Started container csi-snapshotter
  Normal   Pulled           33m                    kubelet            Container image "k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0" already present on machine
  Normal   Created          33m                    kubelet            Created container snapshot-controller
  Normal   Started          33m                    kubelet            Started container snapshot-controller
  Normal   Pulled           33m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0" already present on machine
  Normal   Created          33m                    kubelet            Created container csi-provisioner
  Normal   Started          33m                    kubelet            Started container csi-provisioner
  Normal   Pulled           33m                    kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created          33m                    kubelet            Created container openebs-zfs-plugin
  Normal   Started          33m                    kubelet            Started container openebs-zfs-plugin
  Warning  FailedSync       30m                    kubelet            error determining status: rpc error: code = Unknown desc = Error: No such container: dbc97778beca19a5be9378ee38332432d04fe5612dd687a1cc7227f3553ec061
  Warning  NetworkNotReady  30m (x5 over 30m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   AddedInterface   30m                    multus             Add eth0 [172.16.0.52/16] from ix-net
  Normal   SandboxChanged   30m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   Pulled           30m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-resizer:v1.2.0" already present on machine
  Normal   Created          30m                    kubelet            Created container csi-resizer
  Normal   Started          30m                    kubelet            Started container csi-resizer
  Normal   Pulled           30m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" already present on machine
  Normal   Created          30m                    kubelet            Created container csi-snapshotter
  Normal   Started          30m                    kubelet            Started container csi-snapshotter
  Normal   Pulled           30m                    kubelet            Container image "k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0" already present on machine
  Normal   Created          30m                    kubelet            Created container snapshot-controller
  Normal   Started          30m                    kubelet            Started container snapshot-controller
  Normal   Pulled           30m                    kubelet            Container image "k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0" already present on machine
  Normal   Created          30m                    kubelet            Created container csi-provisioner
  Normal   Started          30m                    kubelet            Started container csi-provisioner
  Normal   Pulled           30m                    kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Started          30m                    kubelet            Started container openebs-zfs-plugin
  Normal   Created          30m                    kubelet            Created container openebs-zfs-plugin
  Warning  NetworkNotReady  2m31s (x6 over 2m40s)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   2m29s                  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   2m28s                  multus             Add eth0 [172.16.0.58/16] from ix-net
  Normal   Pulled           2m28s                  kubelet            Container image "k8s.gcr.io/sig-storage/csi-resizer:v1.2.0" already present on machine
  Normal   Created          2m27s                  kubelet            Created container csi-resizer
  Normal   Started          2m27s                  kubelet            Started container csi-resizer
  Normal   Pulled           2m27s                  kubelet            Container image "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" already present on machine
  Normal   Created          2m26s                  kubelet            Created container csi-snapshotter
  Normal   Started          2m26s                  kubelet            Started container csi-snapshotter
  Normal   Pulled           2m26s                  kubelet            Container image "k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0" already present on machine
  Normal   Created          2m26s                  kubelet            Created container snapshot-controller
  Normal   Started          2m26s                  kubelet            Started container snapshot-controller
  Normal   Pulled           2m26s                  kubelet            Container image "k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0" already present on machine
  Normal   Created          2m25s                  kubelet            Created container csi-provisioner
  Normal   Started          2m24s                  kubelet            Started container csi-provisioner
  Normal   Pulled           2m24s                  kubelet            Container image "openebs/zfs-driver:2.0.0" already present on machine
  Normal   Created          2m24s                  kubelet            Created container openebs-zfs-plugin
  Normal   Started          2m24s                  kubelet            Started container openebs-zfs-plugin


Name:             intel-gpu-plugin-zvqhc
Namespace:        kube-system
Priority:         0
Service Account:  default
Node:             ix-truenas/192.168.2.105
Start Time:       Tue, 11 Apr 2023 14:01:41 +0200
Labels:           app=intel-gpu-plugin
                  controller-revision-hash=847c65dbf9
                  pod-template-generation=1
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.0.57"
                        ],
                        "mac": "ee:d7:61:a4:63:c7",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.0.57
IPs:
  IP:           172.16.0.57
Controlled By:  DaemonSet/intel-gpu-plugin
Init Containers:
  intel-gpu-initcontainer:
    Container ID:   docker://59013bcb2503bf532b4b7f0d2dafeb063e2af34fb39aa64e84b1fa6c81b4bb59
    Image:          intel/intel-gpu-initcontainer:0.19.0
    Image ID:       docker-pullable://intel/intel-gpu-initcontainer@sha256:5f05ec6ff157841189ce7e37fd56708a0e11b4018cc94a003c31724281535fe7
    Port:           <none>
    Host Port:      <none>
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:38:16 +0200
      Finished:     Tue, 11 Apr 2023 14:38:17 +0200
    Ready:          True
    Restart Count:  3
    Environment:    <none>
    Mounts:
      /etc/kubernetes/node-feature-discovery/source.d/ from nfd-source-hooks (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-7f85s (ro)
Containers:
  intel-gpu-plugin:
    Container ID:  docker://48d55de714c1a4c4cc54f6f6c08fd2ab2801b3f40091bad2403dea17876eac87
    Image:         intel/intel-gpu-plugin:0.19.0
    Image ID:      docker-pullable://intel/intel-gpu-plugin@sha256:eaedfe5991710e40ce3bbb530b23f47401a2b60a813ba39439d2fea999aca51c
    Port:          <none>
    Host Port:     <none>
    Args:
      -shared-dev-num
      5
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:19 +0200
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Tue, 11 Apr 2023 14:10:13 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Environment:
      NODE_NAME:   (v1:spec.nodeName)
    Mounts:
      /dev/dri from devfs (ro)
      /sys/class/drm from sysfs (ro)
      /var/lib/kubelet/device-plugins from kubeletsockets (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-7f85s (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  devfs:
    Type:          HostPath (bare host directory volume)
    Path:          /dev/dri
    HostPathType:  
  sysfs:
    Type:          HostPath (bare host directory volume)
    Path:          /sys/class/drm
    HostPathType:  
  kubeletsockets:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/device-plugins
    HostPathType:  
  nfd-source-hooks:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/kubernetes/node-feature-discovery/source.d/
    HostPathType:  DirectoryOrCreate
  kube-api-access-7f85s:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/arch=amd64
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type     Reason           Age                    From               Message
  ----     ------           ----                   ----               -------
  Normal   Scheduled        39m                    default-scheduler  Successfully assigned kube-system/intel-gpu-plugin-zvqhc to ix-truenas
  Normal   AddedInterface   38m                    multus             Add eth0 [172.16.0.43/16] from ix-net
  Normal   Pulled           38m                    kubelet            Container image "intel/intel-gpu-initcontainer:0.19.0" already present on machine
  Normal   Created          38m                    kubelet            Created container intel-gpu-initcontainer
  Normal   Started          38m                    kubelet            Started container intel-gpu-initcontainer
  Normal   Pulled           38m                    kubelet            Container image "intel/intel-gpu-plugin:0.19.0" already present on machine
  Normal   Created          38m                    kubelet            Created container intel-gpu-plugin
  Normal   Started          38m                    kubelet            Started container intel-gpu-plugin
  Warning  NetworkNotReady  33m (x5 over 33m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   33m                    multus             Add eth0 [172.16.0.46/16] from ix-net
  Normal   Pulled           33m                    kubelet            Container image "intel/intel-gpu-initcontainer:0.19.0" already present on machine
  Normal   Created          33m                    kubelet            Created container intel-gpu-initcontainer
  Normal   Started          33m                    kubelet            Started container intel-gpu-initcontainer
  Normal   Pulled           33m                    kubelet            Container image "intel/intel-gpu-plugin:0.19.0" already present on machine
  Normal   Created          33m                    kubelet            Created container intel-gpu-plugin
  Normal   Started          33m                    kubelet            Started container intel-gpu-plugin
  Warning  FailedMount      30m                    kubelet            MountVolume.SetUp failed for volume "kube-api-access-7f85s" : failed to sync configmap cache: timed out waiting for the condition
  Warning  NetworkNotReady  30m (x5 over 30m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   AddedInterface   30m                    multus             Add eth0 [172.16.0.51/16] from ix-net
  Normal   SandboxChanged   30m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   Pulled           30m                    kubelet            Container image "intel/intel-gpu-initcontainer:0.19.0" already present on machine
  Normal   Created          30m                    kubelet            Created container intel-gpu-initcontainer
  Normal   Started          30m                    kubelet            Started container intel-gpu-initcontainer
  Normal   Pulled           30m                    kubelet            Container image "intel/intel-gpu-plugin:0.19.0" already present on machine
  Normal   Created          30m                    kubelet            Created container intel-gpu-plugin
  Normal   Started          30m                    kubelet            Started container intel-gpu-plugin
  Warning  NetworkNotReady  2m31s (x6 over 2m40s)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   2m29s                  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   2m28s                  multus             Add eth0 [172.16.0.57/16] from ix-net
  Normal   Pulled           2m28s                  kubelet            Container image "intel/intel-gpu-initcontainer:0.19.0" already present on machine
  Normal   Created          2m27s                  kubelet            Created container intel-gpu-initcontainer
  Normal   Started          2m27s                  kubelet            Started container intel-gpu-initcontainer
  Normal   Pulled           2m26s                  kubelet            Container image "intel/intel-gpu-plugin:0.19.0" already present on machine
  Normal   Created          2m25s                  kubelet            Created container intel-gpu-plugin
  Normal   Started          2m24s                  kubelet            Started container intel-gpu-plugin


Name:                 coredns-75fc8f8fff-dnkbk
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      coredns
Node:                 ix-truenas/192.168.2.105
Start Time:           Tue, 11 Apr 2023 14:01:41 +0200
Labels:               k8s-app=kube-dns
                      pod-template-hash=75fc8f8fff
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.60"
                            ],
                            "mac": "62:4c:be:94:7c:9c",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.0.60
IPs:
  IP:           172.16.0.60
Controlled By:  ReplicaSet/coredns-75fc8f8fff
Containers:
  coredns:
    Container ID:  docker://465b34c8320e11eb2b3b6f7539e7f54028c627763d59d9599e7be3ede9d670cf
    Image:         rancher/mirrored-coredns-coredns:1.9.1
    Image ID:      docker://sha256:99376d8f35e0abb6ff9d66b50a7c81df6e6dfdb649becc5df84a691a7b4beca4
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:21 +0200
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:10:17 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-574qt (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  custom-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns-custom
    Optional:  true
  kube-api-access-574qt:
    Type:                     Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:   3607
    ConfigMapName:            kube-root-ca.crt
    ConfigMapOptional:        <nil>
    DownwardAPI:              true
QoS Class:                    Burstable
Node-Selectors:               beta.kubernetes.io/os=linux
Tolerations:                  CriticalAddonsOnly op=Exists
                              node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                              node-role.kubernetes.io/master:NoSchedule op=Exists
                              node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                              node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
Events:
  Type     Reason           Age                    From               Message
  ----     ------           ----                   ----               -------
  Normal   Scheduled        39m                    default-scheduler  Successfully assigned kube-system/coredns-75fc8f8fff-dnkbk to ix-truenas
  Normal   AddedInterface   39m                    multus             Add eth0 [172.16.0.39/16] from ix-net
  Normal   Pulled           39m                    kubelet            Container image "rancher/mirrored-coredns-coredns:1.9.1" already present on machine
  Normal   Created          38m                    kubelet            Created container coredns
  Normal   Started          38m                    kubelet            Started container coredns
  Warning  Unhealthy        38m (x2 over 38m)      kubelet            Readiness probe failed: Get "http://172.16.0.39:8181/ready": dial tcp 172.16.0.39:8181: connect: connection refused
  Warning  NetworkNotReady  33m (x5 over 33m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      33m (x5 over 33m)      kubelet            MountVolume.SetUp failed for volume "custom-config-volume" : object "kube-system"/"coredns-custom" not registered
  Warning  FailedMount      33m (x5 over 33m)      kubelet            MountVolume.SetUp failed for volume "config-volume" : object "kube-system"/"coredns" not registered
  Normal   SandboxChanged   33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   33m                    multus             Add eth0 [172.16.0.47/16] from ix-net
  Normal   Pulled           33m                    kubelet            Container image "rancher/mirrored-coredns-coredns:1.9.1" already present on machine
  Normal   Created          33m                    kubelet            Created container coredns
  Normal   Started          33m                    kubelet            Started container coredns
  Warning  Unhealthy        33m                    kubelet            Readiness probe failed: Get "http://172.16.0.47:8181/ready": dial tcp 172.16.0.47:8181: connect: connection refused
  Warning  FailedMount      30m                    kubelet            MountVolume.SetUp failed for volume "kube-api-access-574qt" : failed to sync configmap cache: timed out waiting for the condition
  Warning  FailedMount      30m (x5 over 30m)      kubelet            MountVolume.SetUp failed for volume "config-volume" : object "kube-system"/"coredns" not registered
  Warning  FailedMount      30m (x5 over 30m)      kubelet            MountVolume.SetUp failed for volume "custom-config-volume" : object "kube-system"/"coredns-custom" not registered
  Warning  NetworkNotReady  30m (x6 over 30m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   30m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   30m                    multus             Add eth0 [172.16.0.53/16] from ix-net
  Normal   Pulled           30m                    kubelet            Container image "rancher/mirrored-coredns-coredns:1.9.1" already present on machine
  Normal   Created          30m                    kubelet            Created container coredns
  Normal   Started          30m                    kubelet            Started container coredns
  Warning  Unhealthy        30m (x3 over 30m)      kubelet            Readiness probe failed: Get "http://172.16.0.53:8181/ready": dial tcp 172.16.0.53:8181: connect: connection refused
  Warning  FailedMount      2m32s (x5 over 2m40s)  kubelet            MountVolume.SetUp failed for volume "custom-config-volume" : object "kube-system"/"coredns-custom" not registered
  Warning  FailedMount      2m32s (x5 over 2m40s)  kubelet            MountVolume.SetUp failed for volume "config-volume" : object "kube-system"/"coredns" not registered
  Warning  NetworkNotReady  2m31s (x6 over 2m40s)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   2m24s                  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   2m23s                  multus             Add eth0 [172.16.0.60/16] from ix-net
  Normal   Pulled           2m23s                  kubelet            Container image "rancher/mirrored-coredns-coredns:1.9.1" already present on machine
  Normal   Created          2m22s                  kubelet            Created container coredns
  Normal   Started          2m22s                  kubelet            Started container coredns
  Warning  Unhealthy        2m21s (x2 over 2m22s)  kubelet            Readiness probe failed: Get "http://172.16.0.60:8181/ready": dial tcp 172.16.0.60:8181: connect: connection refused


Name:             prometheus-operator-5dcffb7cb8-q6xxq
Namespace:        prometheus-operator
Priority:         0
Service Account:  prometheus-operator
Node:             ix-truenas/192.168.2.105
Start Time:       Tue, 11 Apr 2023 14:01:41 +0200
Labels:           app.kubernetes.io/component=controller
                  app.kubernetes.io/name=prometheus-operator
                  app.kubernetes.io/version=0.60.1
                  pod-template-hash=5dcffb7cb8
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.0.61"
                        ],
                        "mac": "ba:8d:b6:5f:dd:a7",
                        "default": true,
                        "dns": {}
                    }]
                  kubectl.kubernetes.io/default-container: prometheus-operator
Status:           Running
IP:               172.16.0.61
IPs:
  IP:           172.16.0.61
Controlled By:  ReplicaSet/prometheus-operator-5dcffb7cb8
Containers:
  prometheus-operator:
    Container ID:  docker://2b728c7fcb18851e757ab0f4945f7a65a351cfa194aa817f5edd9ac3f2d6aa1a
    Image:         quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Image ID:      docker-pullable://quay.io/prometheus-operator/prometheus-operator@sha256:63d4d0f8fb817313ec13c2960b72d1b4341bea269eb97ec7bf44c585a04b1e2e
    Port:          8080/TCP
    Host Port:     0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:22 +0200
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:10:18 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-sphlt (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-sphlt:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason           Age                    From               Message
  ----     ------           ----                   ----               -------
  Normal   Scheduled        39m                    default-scheduler  Successfully assigned prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq to ix-truenas
  Normal   AddedInterface   39m                    multus             Add eth0 [172.16.0.40/16] from ix-net
  Normal   Pulled           39m                    kubelet            Container image "quay.io/prometheus-operator/prometheus-operator:v0.60.1" already present on machine
  Normal   Created          38m                    kubelet            Created container prometheus-operator
  Normal   Started          38m                    kubelet            Started container prometheus-operator
  Warning  NetworkNotReady  33m (x5 over 33m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      33m (x5 over 33m)      kubelet            MountVolume.SetUp failed for volume "kube-api-access-sphlt" : object "prometheus-operator"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   33m                    multus             Add eth0 [172.16.0.50/16] from ix-net
  Normal   Pulled           33m                    kubelet            Container image "quay.io/prometheus-operator/prometheus-operator:v0.60.1" already present on machine
  Normal   Created          33m                    kubelet            Created container prometheus-operator
  Normal   Started          33m                    kubelet            Started container prometheus-operator
  Warning  NetworkNotReady  30m (x5 over 30m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      30m (x5 over 30m)      kubelet            MountVolume.SetUp failed for volume "kube-api-access-sphlt" : object "prometheus-operator"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   30m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   30m                    multus             Add eth0 [172.16.0.55/16] from ix-net
  Normal   Pulled           30m                    kubelet            Container image "quay.io/prometheus-operator/prometheus-operator:v0.60.1" already present on machine
  Normal   Created          30m                    kubelet            Created container prometheus-operator
  Normal   Started          30m                    kubelet            Started container prometheus-operator
  Warning  NetworkNotReady  2m31s (x6 over 2m40s)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      2m31s (x5 over 2m38s)  kubelet            MountVolume.SetUp failed for volume "kube-api-access-sphlt" : object "prometheus-operator"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   2m23s                  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   2m22s                  multus             Add eth0 [172.16.0.61/16] from ix-net
  Normal   Pulled           2m22s                  kubelet            Container image "quay.io/prometheus-operator/prometheus-operator:v0.60.1" already present on machine
  Normal   Created          2m21s                  kubelet            Created container prometheus-operator
  Normal   Started          2m21s                  kubelet            Started container prometheus-operator


Name:             controller-84d6d4db45-2jc2g
Namespace:        metallb-system
Priority:         0
Service Account:  controller
Node:             ix-truenas/192.168.2.105
Start Time:       Tue, 11 Apr 2023 14:01:41 +0200
Labels:           app=metallb
                  component=controller
                  pod-template-hash=84d6d4db45
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.0.59"
                        ],
                        "mac": "12:f2:1a:3a:04:d6",
                        "default": true,
                        "dns": {}
                    }]
                  prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Running
IP:               172.16.0.59
IPs:
  IP:           172.16.0.59
Controlled By:  ReplicaSet/controller-84d6d4db45
Containers:
  controller:
    Container ID:  docker://b13c29bb3462f2b70a97ea728ab0ac887250102a0633e84817c33de19fe40579
    Image:         quay.io/metallb/controller:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/controller@sha256:0e486995b27d62433e728f7b266fb8b7b7f4fd65e1b751bef17fe56e76653bf1
    Ports:         7472/TCP, 9443/TCP
    Host Ports:    0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:21 +0200
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:10:17 +0200
      Finished:     Tue, 11 Apr 2023 14:14:37 +0200
    Ready:          True
    Restart Count:  3
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-8k2r4 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
  kube-api-access-8k2r4:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason           Age                    From               Message
  ----     ------           ----                   ----               -------
  Normal   Scheduled        39m                    default-scheduler  Successfully assigned metallb-system/controller-84d6d4db45-2jc2g to ix-truenas
  Normal   AddedInterface   38m                    multus             Add eth0 [172.16.0.41/16] from ix-net
  Normal   Pulled           38m                    kubelet            Container image "quay.io/metallb/controller:v0.13.7" already present on machine
  Normal   Created          38m                    kubelet            Created container controller
  Normal   Started          38m                    kubelet            Started container controller
  Warning  NetworkNotReady  33m (x5 over 33m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      33m (x5 over 33m)      kubelet            MountVolume.SetUp failed for volume "cert" : object "metallb-system"/"webhook-server-cert" not registered
  Normal   SandboxChanged   33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   33m                    multus             Add eth0 [172.16.0.48/16] from ix-net
  Normal   Pulled           33m                    kubelet            Container image "quay.io/metallb/controller:v0.13.7" already present on machine
  Normal   Created          33m                    kubelet            Created container controller
  Normal   Started          33m                    kubelet            Started container controller
  Warning  NetworkNotReady  30m (x5 over 30m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      30m (x5 over 30m)      kubelet            MountVolume.SetUp failed for volume "cert" : object "metallb-system"/"webhook-server-cert" not registered
  Normal   SandboxChanged   30m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   30m                    multus             Add eth0 [172.16.0.54/16] from ix-net
  Normal   Pulled           30m                    kubelet            Container image "quay.io/metallb/controller:v0.13.7" already present on machine
  Normal   Created          30m                    kubelet            Created container controller
  Normal   Started          30m                    kubelet            Started container controller
  Warning  FailedMount      2m32s (x5 over 2m40s)  kubelet            MountVolume.SetUp failed for volume "cert" : object "metallb-system"/"webhook-server-cert" not registered
  Warning  NetworkNotReady  2m31s (x6 over 2m40s)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Normal   SandboxChanged   2m24s                  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   2m23s                  multus             Add eth0 [172.16.0.59/16] from ix-net
  Normal   Pulled           2m23s                  kubelet            Container image "quay.io/metallb/controller:v0.13.7" already present on machine
  Normal   Created          2m22s                  kubelet            Created container controller
  Normal   Started          2m22s                  kubelet            Started container controller


Name:             netdata-7bc8fbb7b4-8vz7s
Namespace:        ix-netdata
Priority:         0
Service Account:  netdata
Node:             ix-truenas/192.168.2.105
Start Time:       Tue, 11 Apr 2023 14:01:41 +0200
Labels:           app.kubernetes.io/instance=netdata
                  app.kubernetes.io/name=netdata
                  pod-template-hash=7bc8fbb7b4
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.0.62"
                        ],
                        "mac": "56:81:15:b7:6d:d3",
                        "default": true,
                        "dns": {}
                    }]
                  rollme: 5Tu4r
Status:           Running
IP:               172.16.0.62
IPs:
  IP:           172.16.0.62
Controlled By:  ReplicaSet/netdata-7bc8fbb7b4
Containers:
  netdata:
    Container ID:   docker://92e8ed3ab572e3efc34ce61dc3b6bd91c1b3cf95ac4e809e3c7dac8dcf3ed767
    Image:          netdata/netdata:v1.38.1
    Image ID:       docker-pullable://netdata/netdata@sha256:a85154c3c1aa28e5dd7a22974db2332cee03f834bc5e84be4d9283d8acc2c1c9
    Port:           19999/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Tue, 11 Apr 2023 14:38:26 +0200
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 11 Apr 2023 14:10:21 +0200
      Finished:     Tue, 11 Apr 2023 14:14:44 +0200
    Ready:          True
    Restart Count:  3
    Liveness:       http-get http://:http/api/v1/info delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:      http-get http://:http/api/v1/info delay=10s timeout=5s period=10s #success=2 #failure=5
    Startup:        http-get http://:http/api/v1/info delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:    <none>
    Mounts:
      /etc/netdata from netdataconfig (rw)
      /host/etc/group from group (ro)
      /host/etc/os-release from os-release (ro)
      /host/etc/passwd from user (ro)
      /host/proc from proc (ro)
      /host/sys from sys (ro)
      /var/cache/netdata from netdatacache (rw)
      /var/lib/netdata from netdatalib (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-gd6g7 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  netdatacache:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-cache
    HostPathType:  
  netdataconfig:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-config
    HostPathType:  
  netdatalib:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/RaidZ1/ix-applications/releases/netdata/volumes/ix_volumes/ix-lib
    HostPathType:  
  os-release:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/os-release
    HostPathType:  
  proc:
    Type:          HostPath (bare host directory volume)
    Path:          /proc
    HostPathType:  
  sys:
    Type:          HostPath (bare host directory volume)
    Path:          /sys
    HostPathType:  
  user:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/passwd
    HostPathType:  
  group:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/group
    HostPathType:  
  kube-api-access-gd6g7:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason           Age                    From               Message
  ----     ------           ----                   ----               -------
  Normal   Scheduled        39m                    default-scheduler  Successfully assigned ix-netdata/netdata-7bc8fbb7b4-8vz7s to ix-truenas
  Normal   AddedInterface   38m                    multus             Add eth0 [172.16.0.42/16] from ix-net
  Normal   Pulled           38m                    kubelet            Container image "netdata/netdata:v1.38.1" already present on machine
  Normal   Created          38m                    kubelet            Created container netdata
  Normal   Started          38m                    kubelet            Started container netdata
  Warning  NetworkNotReady  33m (x5 over 33m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      33m (x5 over 33m)      kubelet            MountVolume.SetUp failed for volume "kube-api-access-gd6g7" : object "ix-netdata"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   33m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   33m                    multus             Add eth0 [172.16.0.49/16] from ix-net
  Normal   Pulled           33m                    kubelet            Container image "netdata/netdata:v1.38.1" already present on machine
  Normal   Created          33m                    kubelet            Created container netdata
  Normal   Started          33m                    kubelet            Started container netdata
  Warning  NetworkNotReady  30m (x5 over 30m)      kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      30m (x5 over 30m)      kubelet            MountVolume.SetUp failed for volume "kube-api-access-gd6g7" : object "ix-netdata"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   30m                    kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   30m                    multus             Add eth0 [172.16.0.56/16] from ix-net
  Normal   Pulled           30m                    kubelet            Container image "netdata/netdata:v1.38.1" already present on machine
  Normal   Created          30m                    kubelet            Created container netdata
  Normal   Started          30m                    kubelet            Started container netdata
  Warning  NetworkNotReady  2m31s (x6 over 2m40s)  kubelet            network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
  Warning  FailedMount      2m30s (x5 over 2m38s)  kubelet            MountVolume.SetUp failed for volume "kube-api-access-gd6g7" : object "ix-netdata"/"kube-root-ca.crt" not registered
  Normal   SandboxChanged   2m22s                  kubelet            Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface   2m21s                  multus             Add eth0 [172.16.0.62/16] from ix-net
  Normal   Pulled           2m21s                  kubelet            Container image "netdata/netdata:v1.38.1" already present on machine
  Normal   Created          2m17s                  kubelet            Created container netdata
  Normal   Started          2m17s                  kubelet            Started container netdata
debug finished in 0 seconds for k3s kubectl describe pods --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe services --all-namespaces @1681216843           +
+--------------------------------------------------------------------------------+
Name:              kubernetes
Namespace:         default
Labels:            component=apiserver
                   provider=kubernetes
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.1
IPs:               172.17.0.1
Port:              https  443/TCP
TargetPort:        6443/TCP
Endpoints:         192.168.2.105:6443
Session Affinity:  None
Events:            <none>


Name:              kube-dns
Namespace:         kube-system
Labels:            k8s-app=kube-dns
                   kubernetes.io/cluster-service=true
                   kubernetes.io/name=CoreDNS
                   objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:       objectset.rio.cattle.io/applied:
                     H4sIAAAAAAAA/4yRwW7UMBCGXwXN2Qlxstukljig9oKQUKUCF8TBcWZZk8S2PJMgtMq7I++mYqEK9JZoPn/6558T6GA/YyTrHSiYJQjoretAwSPG2RoEASOy7jRrUCfQznnWbL2j9O...
                   objectset.rio.cattle.io/id: 
                   objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                   objectset.rio.cattle.io/owner-name: coredns
                   objectset.rio.cattle.io/owner-namespace: kube-system
                   prometheus.io/port: 9153
                   prometheus.io/scrape: true
Selector:          k8s-app=kube-dns
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.10
IPs:               172.17.0.10
Port:              dns  53/UDP
TargetPort:        53/UDP
Endpoints:         172.16.0.60:53
Port:              dns-tcp  53/TCP
TargetPort:        53/TCP
Endpoints:         172.16.0.60:53
Port:              metrics  9153/TCP
TargetPort:        9153/TCP
Endpoints:         172.16.0.60:9153
Session Affinity:  None
Events:            <none>


Name:              webhook-service
Namespace:         metallb-system
Labels:            <none>
Annotations:       <none>
Selector:          component=controller
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.209.192
IPs:               172.17.209.192
Port:              <unset>  443/TCP
TargetPort:        9443/TCP
Endpoints:         172.16.0.59:9443
Session Affinity:  None
Events:            <none>


Name:              prometheus-operator
Namespace:         prometheus-operator
Labels:            app.kubernetes.io/component=controller
                   app.kubernetes.io/name=prometheus-operator
                   app.kubernetes.io/version=0.60.1
Annotations:       <none>
Selector:          app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                None
IPs:               None
Port:              http  8080/TCP
TargetPort:        http/TCP
Endpoints:         172.16.0.61:8080
Session Affinity:  None
Events:            <none>


Name:              kubelet
Namespace:         kube-system
Labels:            app.kubernetes.io/managed-by=prometheus-operator
                   app.kubernetes.io/name=kubelet
                   k8s-app=kubelet
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  RequireDualStack
IP Families:       IPv4,IPv6
IP:                None
IPs:               None
Port:              https-metrics  10250/TCP
TargetPort:        10250/TCP
Endpoints:         192.168.2.105:10250
Port:              http-metrics  10255/TCP
TargetPort:        10255/TCP
Endpoints:         192.168.2.105:10255
Port:              cadvisor  4194/TCP
TargetPort:        4194/TCP
Endpoints:         192.168.2.105:4194
Session Affinity:  None
Events:            <none>


Name:                     netdata
Namespace:                ix-netdata
Labels:                   app.kubernetes.io/instance=netdata
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=netdata
                          app.kubernetes.io/version=v1.38.1
                          helm.sh/chart=netdata-1.0.16
Annotations:              meta.helm.sh/release-name: netdata
                          meta.helm.sh/release-namespace: ix-netdata
Selector:                 app.kubernetes.io/instance=netdata,app.kubernetes.io/name=netdata
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.55.206
IPs:                      172.17.55.206
Port:                     ui  20489/TCP
TargetPort:               19999/TCP
NodePort:                 ui  20489/TCP
Endpoints:                172.16.0.62:19999
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>
debug finished in 0 seconds for k3s kubectl describe services --all-namespaces


+--------------------------------------------------------------------------------+
+         k3s kubectl describe statefulset --all-namespaces @1681216843          +
+--------------------------------------------------------------------------------+
Name:               openebs-zfs-controller
Namespace:          kube-system
CreationTimestamp:  Mon, 09 Jan 2023 20:54:07 +0100
Selector:           app=openebs-zfs-controller,role=openebs-zfs
Labels:             objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
Annotations:        objectset.rio.cattle.io/applied:
                      H4sIAAAAAAAA/8xWXY/aOBT9KyurD60U8zEzu9ONxAMdshLqFBCgSlWFkHFughfH9to3Wego/33lBJh0Zpgvdau+ge17fM7xuY5vCDPiM1gntCIhYca4dtElAdkIFZOQzJAhJLmcAZ...
                    objectset.rio.cattle.io/id: 
                    objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                    objectset.rio.cattle.io/owner-name: zfs-operator
                    objectset.rio.cattle.io/owner-namespace: kube-system
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
  Partition:        0
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-controller
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-controller-sa
  Containers:
   csi-resizer:
    Image:      k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   csi-snapshotter:
    Image:      k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   snapshot-controller:
    Image:      k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --leader-election=true
    Environment:  <none>
    Mounts:       <none>
   csi-provisioner:
    Image:      k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:   (v1:metadata.namespace)
      POD_NAME:    (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
  Volumes:
   socket-dir:
    Type:               EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:             
    SizeLimit:          <unset>
  Priority Class Name:  openebs-zfs-csi-controller-critical
Volume Claims:          <none>
Events:
  Type     Reason               Age   From                    Message
  ----     ------               ----  ----                    -------
  Warning  RecreatingFailedPod  80m   statefulset-controller  StatefulSet kube-system/openebs-zfs-controller is recreating failed Pod openebs-zfs-controller-0
  Normal   SuccessfulDelete     80m   statefulset-controller  delete Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful
  Normal   SuccessfulCreate     80m   statefulset-controller  create Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful
  Warning  RecreatingFailedPod  69m   statefulset-controller  StatefulSet kube-system/openebs-zfs-controller is recreating failed Pod openebs-zfs-controller-0
  Normal   SuccessfulDelete     69m   statefulset-controller  delete Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful
  Normal   SuccessfulCreate     69m   statefulset-controller  create Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful
  Warning  RecreatingFailedPod  39m   statefulset-controller  StatefulSet kube-system/openebs-zfs-controller is recreating failed Pod openebs-zfs-controller-0
  Normal   SuccessfulDelete     39m   statefulset-controller  delete Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful
  Normal   SuccessfulCreate     39m   statefulset-controller  create Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful
debug finished in 0 seconds for k3s kubectl describe statefulset --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe job --all-namespaces @1681216843              +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 0 seconds for k3s kubectl describe job --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe cronjob --all-namespaces @1681216843            +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 0 seconds for k3s kubectl describe cronjob --all-namespaces


+--------------------------------------------------------------------------------+
+          k3s kubectl describe daemonsets --all-namespaces @1681216843          +
+--------------------------------------------------------------------------------+
Name:           openebs-zfs-node
Selector:       app=openebs-zfs-node
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                openebs.io/component-name=openebs-zfs-node
                openebs.io/version=ci
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIezSdnuR9oFCrl11FxBsew/VCjn2JPhw7Mh20uVW+e8nOwEChO2e1Lcknvlm5ptvJn5COGffQWkmBQoRznMdlBfIQ2smKArRGEMmxQIM8l...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                objectset.rio.cattle.io/owner-name: zfs-operator
                objectset.rio.cattle.io/owner-namespace: kube-system
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-node
                    openebs.io/component-name=openebs-zfs-node
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-node-sa
  Containers:
   csi-node-driver-registrar:
    Image:      k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
  Volumes:
   device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
   encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
   chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
   host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
   registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
   plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
   pods-mount-dir:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/
    HostPathType:       Directory
  Priority Class Name:  openebs-zfs-csi-node-critical
Events:
  Type     Reason            Age   From                  Message
  ----     ------            ----  ----                  -------
  Normal   SuccessfulCreate  80m   daemonset-controller  Created pod: openebs-zfs-node-szq7f
  Warning  FailedDaemonPod   69m   daemonset-controller  Found failed daemon pod kube-system/openebs-zfs-node-szq7f on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  69m   daemonset-controller  Deleted pod: openebs-zfs-node-szq7f
  Normal   SuccessfulCreate  69m   daemonset-controller  Created pod: openebs-zfs-node-65rml
  Warning  FailedDaemonPod   39m   daemonset-controller  Found failed daemon pod kube-system/openebs-zfs-node-65rml on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  39m   daemonset-controller  Deleted pod: openebs-zfs-node-65rml
  Normal   SuccessfulCreate  39m   daemonset-controller  Created pod: openebs-zfs-node-bwxzr


Name:           speaker
Selector:       app=metallb,component=speaker
Node-Selector:  kubernetes.io/os=linux
Labels:         app=metallb
                component=speaker
Annotations:    deprecated.daemonset.template.generation: 1
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=metallb
                    component=speaker
  Annotations:      prometheus.io/port: 7472
                    prometheus.io/scrape: true
  Service Account:  speaker
  Containers:
   speaker:
    Image:       quay.io/metallb/speaker:v0.13.7
    Ports:       7472/TCP, 7946/TCP, 7946/UDP
    Host Ports:  7472/TCP, 7946/TCP, 7946/UDP
    Args:
      --port=7472
      --log-level=info
    Liveness:   http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_NODE_NAME:       (v1:spec.nodeName)
      METALLB_HOST:            (v1:status.hostIP)
      METALLB_ML_BIND_ADDR:    (v1:status.podIP)
      METALLB_ML_LABELS:      app=metallb,component=speaker
      METALLB_ML_SECRET_KEY:  <set to the key 'secretkey' in secret 'memberlist'>  Optional: false
    Mounts:                   <none>
  Volumes:                    <none>
Events:
  Type     Reason            Age   From                  Message
  ----     ------            ----  ----                  -------
  Warning  FailedDaemonPod   80m   daemonset-controller  Found failed daemon pod metallb-system/speaker-5wdpn on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  80m   daemonset-controller  Deleted pod: speaker-5wdpn
  Normal   SuccessfulCreate  80m   daemonset-controller  Created pod: speaker-f594j
  Warning  FailedDaemonPod   69m   daemonset-controller  Found failed daemon pod metallb-system/speaker-f594j on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  69m   daemonset-controller  Deleted pod: speaker-f594j
  Normal   SuccessfulCreate  69m   daemonset-controller  Created pod: speaker-wsqv9
  Warning  FailedDaemonPod   39m   daemonset-controller  Found failed daemon pod metallb-system/speaker-wsqv9 on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  39m   daemonset-controller  Deleted pod: speaker-wsqv9
  Normal   SuccessfulCreate  39m   daemonset-controller  Created pod: speaker-rcq54


Name:           intel-gpu-plugin
Selector:       app=intel-gpu-plugin
Node-Selector:  kubernetes.io/arch=amd64
Labels:         app=intel-gpu-plugin
Annotations:    deprecated.daemonset.template.generation: 1
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:  app=intel-gpu-plugin
  Init Containers:
   intel-gpu-initcontainer:
    Image:        intel/intel-gpu-initcontainer:0.19.0
    Port:         <none>
    Host Port:    <none>
    Environment:  <none>
    Mounts:
      /etc/kubernetes/node-feature-discovery/source.d/ from nfd-source-hooks (rw)
  Containers:
   intel-gpu-plugin:
    Image:      intel/intel-gpu-plugin:0.19.0
    Port:       <none>
    Host Port:  <none>
    Args:
      -shared-dev-num
      5
    Environment:
      NODE_NAME:   (v1:spec.nodeName)
    Mounts:
      /dev/dri from devfs (ro)
      /sys/class/drm from sysfs (ro)
      /var/lib/kubelet/device-plugins from kubeletsockets (rw)
  Volumes:
   devfs:
    Type:          HostPath (bare host directory volume)
    Path:          /dev/dri
    HostPathType:  
   sysfs:
    Type:          HostPath (bare host directory volume)
    Path:          /sys/class/drm
    HostPathType:  
   kubeletsockets:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/device-plugins
    HostPathType:  
   nfd-source-hooks:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/kubernetes/node-feature-discovery/source.d/
    HostPathType:  DirectoryOrCreate
Events:
  Type     Reason            Age   From                  Message
  ----     ------            ----  ----                  -------
  Warning  FailedDaemonPod   80m   daemonset-controller  Found failed daemon pod kube-system/intel-gpu-plugin-mhpnn on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  80m   daemonset-controller  Deleted pod: intel-gpu-plugin-mhpnn
  Normal   SuccessfulCreate  80m   daemonset-controller  Created pod: intel-gpu-plugin-mthvp
  Warning  FailedDaemonPod   69m   daemonset-controller  Found failed daemon pod kube-system/intel-gpu-plugin-mthvp on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  69m   daemonset-controller  Deleted pod: intel-gpu-plugin-mthvp
  Normal   SuccessfulCreate  69m   daemonset-controller  Created pod: intel-gpu-plugin-bcx5x
  Warning  FailedDaemonPod   39m   daemonset-controller  Found failed daemon pod kube-system/intel-gpu-plugin-bcx5x on node ix-truenas, will try to kill it
  Normal   SuccessfulDelete  39m   daemonset-controller  Deleted pod: intel-gpu-plugin-bcx5x
  Normal   SuccessfulCreate  39m   daemonset-controller  Created pod: intel-gpu-plugin-zvqhc
debug finished in 0 seconds for k3s kubectl describe daemonsets --all-namespaces


+--------------------------------------------------------------------------------+
+                            docker ps -a @1681216843                            +
+--------------------------------------------------------------------------------+
CONTAINER ID   IMAGE                        COMMAND                  CREATED          STATUS                        PORTS     NAMES
92e8ed3ab572   3b89e89cfe47                 "/usr/sbin/run.sh"       2 minutes ago    Up 2 minutes                            k8s_netdata_netdata-7bc8fbb7b4-8vz7s_ix-netdata_c4438af0-7353-463f-9a23-1d1bd583a23d_3
2b728c7fcb18   df968278084b                 "/bin/operator --kub…"   2 minutes ago    Up 2 minutes                            k8s_prometheus-operator_prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator_c0561815-b5dd-4748-b1d5-fb362bcd5682_3
10222db1251f   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_netdata-7bc8fbb7b4-8vz7s_ix-netdata_c4438af0-7353-463f-9a23-1d1bd583a23d_3
83da8920ea90   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator_c0561815-b5dd-4748-b1d5-fb362bcd5682_3
465b34c8320e   99376d8f35e0                 "/coredns -conf /etc…"   2 minutes ago    Up 2 minutes                            k8s_coredns_coredns-75fc8f8fff-dnkbk_kube-system_30a4863f-b5c5-44ac-bd90-df544fd65f1b_3
b13c29bb3462   e73361dabfb8                 "/controller --port=…"   2 minutes ago    Up 2 minutes                            k8s_controller_controller-84d6d4db45-2jc2g_metallb-system_24978195-ae80-42c9-80e3-6e880be2bac2_3
5989dc107071   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_controller-84d6d4db45-2jc2g_metallb-system_24978195-ae80-42c9-80e3-6e880be2bac2_3
d2673f5397c6   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_coredns-75fc8f8fff-dnkbk_kube-system_30a4863f-b5c5-44ac-bd90-df544fd65f1b_3
42f866aca9ed   2e17748d7c44                 "/usr/local/bin/zfs-…"   2 minutes ago    Up 2 minutes                            k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_3
6001b6e8dd43   fe0f921f3c92                 "/csi-provisioner --…"   2 minutes ago    Up 2 minutes                            k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_3
48d55de714c1   f2c60b24fad8                 "/usr/local/bin/inte…"   2 minutes ago    Up 2 minutes                            k8s_intel-gpu-plugin_intel-gpu-plugin-zvqhc_kube-system_ee2f204d-7d95-4555-b933-b6ba87ba1547_3
8704603c35d3   f1d8a00ae690                 "/snapshot-controlle…"   2 minutes ago    Up 2 minutes                            k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_3
7a729ff831b4   da32a49a903a                 "/csi-snapshotter --…"   2 minutes ago    Up 2 minutes                            k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_3
f0a2eafdda3b   0aa9629e1508                 "/csi-resizer --v=5 …"   2 minutes ago    Up 2 minutes                            k8s_csi-resizer_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_3
59013bcb2503   e026ae0f49c4                 "/usr/local/bin/gpu-…"   2 minutes ago    Exited (0) 2 minutes ago                k8s_intel-gpu-initcontainer_intel-gpu-plugin-zvqhc_kube-system_ee2f204d-7d95-4555-b933-b6ba87ba1547_3
45750e97d579   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_intel-gpu-plugin-zvqhc_kube-system_ee2f204d-7d95-4555-b933-b6ba87ba1547_3
9741c6362663   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_3
fc0253809d91   2e17748d7c44                 "/usr/local/bin/zfs-…"   2 minutes ago    Up 2 minutes                            k8s_openebs-zfs-plugin_openebs-zfs-node-bwxzr_kube-system_2a0727c6-327f-47a2-8bb9-c8741fd9f6ed_3
feddfb1a978e   738c5d221d60                 "/speaker --port=747…"   2 minutes ago    Up 2 minutes                            k8s_speaker_speaker-rcq54_metallb-system_c1e17754-0080-4f9e-9fc0-4a63b0e92e6e_3
0b3856f9340c   368ee7d6e60f                 "/csi-node-driver-re…"   2 minutes ago    Up 2 minutes                            k8s_csi-node-driver-registrar_openebs-zfs-node-bwxzr_kube-system_2a0727c6-327f-47a2-8bb9-c8741fd9f6ed_3
4fdf5551e237   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_speaker-rcq54_metallb-system_c1e17754-0080-4f9e-9fc0-4a63b0e92e6e_3
16ccd5a63c8f   rancher/mirrored-pause:3.6   "/pause"                 2 minutes ago    Up 2 minutes                            k8s_POD_openebs-zfs-node-bwxzr_kube-system_2a0727c6-327f-47a2-8bb9-c8741fd9f6ed_3
ac6914b3cb90   3b89e89cfe47                 "/usr/sbin/run.sh"       30 minutes ago   Exited (0) 25 minutes ago               k8s_netdata_netdata-7bc8fbb7b4-8vz7s_ix-netdata_c4438af0-7353-463f-9a23-1d1bd583a23d_2
2e55de6a8db0   df968278084b                 "/bin/operator --kub…"   30 minutes ago   Exited (0) 26 minutes ago               k8s_prometheus-operator_prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator_c0561815-b5dd-4748-b1d5-fb362bcd5682_2
083141b01aa9   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_netdata-7bc8fbb7b4-8vz7s_ix-netdata_c4438af0-7353-463f-9a23-1d1bd583a23d_2
afb1c5851518   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator_c0561815-b5dd-4748-b1d5-fb362bcd5682_2
294128e0cff0   e73361dabfb8                 "/controller --port=…"   30 minutes ago   Exited (0) 26 minutes ago               k8s_controller_controller-84d6d4db45-2jc2g_metallb-system_24978195-ae80-42c9-80e3-6e880be2bac2_2
9bc27475f33c   99376d8f35e0                 "/coredns -conf /etc…"   30 minutes ago   Exited (0) 26 minutes ago               k8s_coredns_coredns-75fc8f8fff-dnkbk_kube-system_30a4863f-b5c5-44ac-bd90-df544fd65f1b_2
23508512b7c2   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_coredns-75fc8f8fff-dnkbk_kube-system_30a4863f-b5c5-44ac-bd90-df544fd65f1b_2
242211d17979   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_controller-84d6d4db45-2jc2g_metallb-system_24978195-ae80-42c9-80e3-6e880be2bac2_2
82cdbeb28966   2e17748d7c44                 "/usr/local/bin/zfs-…"   30 minutes ago   Exited (137) 25 minutes ago             k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_2
938c74f370a1   fe0f921f3c92                 "/csi-provisioner --…"   30 minutes ago   Exited (2) 26 minutes ago               k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_2
7606d7e4c9fb   f2c60b24fad8                 "/usr/local/bin/inte…"   30 minutes ago   Exited (2) 26 minutes ago               k8s_intel-gpu-plugin_intel-gpu-plugin-zvqhc_kube-system_ee2f204d-7d95-4555-b933-b6ba87ba1547_2
f92ba0fb7ec2   f1d8a00ae690                 "/snapshot-controlle…"   30 minutes ago   Exited (2) 26 minutes ago               k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_2
612cc9f67887   da32a49a903a                 "/csi-snapshotter --…"   30 minutes ago   Exited (2) 26 minutes ago               k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_2
00409368a4a4   0aa9629e1508                 "/csi-resizer --v=5 …"   30 minutes ago   Exited (2) 26 minutes ago               k8s_csi-resizer_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_2
aad36eaac89b   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_openebs-zfs-controller-0_kube-system_d03d171f-7cf6-4f50-878b-5cb4e9023b3e_2
f6851aee517b   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_intel-gpu-plugin-zvqhc_kube-system_ee2f204d-7d95-4555-b933-b6ba87ba1547_2
8eed371ba90d   2e17748d7c44                 "/usr/local/bin/zfs-…"   30 minutes ago   Exited (137) 25 minutes ago             k8s_openebs-zfs-plugin_openebs-zfs-node-bwxzr_kube-system_2a0727c6-327f-47a2-8bb9-c8741fd9f6ed_2
b78033c420f1   368ee7d6e60f                 "/csi-node-driver-re…"   30 minutes ago   Exited (0) 26 minutes ago               k8s_csi-node-driver-registrar_openebs-zfs-node-bwxzr_kube-system_2a0727c6-327f-47a2-8bb9-c8741fd9f6ed_2
25a463805bb0   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_openebs-zfs-node-bwxzr_kube-system_2a0727c6-327f-47a2-8bb9-c8741fd9f6ed_2
96cc72820079   738c5d221d60                 "/speaker --port=747…"   30 minutes ago   Exited (0) 26 minutes ago               k8s_speaker_speaker-rcq54_metallb-system_c1e17754-0080-4f9e-9fc0-4a63b0e92e6e_2
cde74b38e823   rancher/mirrored-pause:3.6   "/pause"                 30 minutes ago   Exited (0) 26 minutes ago               k8s_POD_speaker-rcq54_metallb-system_c1e17754-0080-4f9e-9fc0-4a63b0e92e6e_2
debug finished in 0 seconds for docker ps -a


+--------------------------------------------------------------------------------+
+                          docker images -a @1681216843                          +
+--------------------------------------------------------------------------------+
REPOSITORY                                         TAG                          IMAGE ID       CREATED         SIZE
alpine                                             latest                       9ed4aefc74f6   12 days ago     7.05MB
netdata/netdata                                    v1.38.1                      3b89e89cfe47   8 weeks ago     394MB
alpine                                             <none>                       042a816809aa   3 months ago    7.05MB
tccr.io/truecharts/storj-node                      <none>                       96aa6989f7cf   3 months ago    124MB
tccr.io/truecharts/kubectl                         <none>                       672c44cd9781   3 months ago    117MB
tccr.io/truecharts/alpine                          <none>                       f34bf968962a   3 months ago    10.2MB
rancher/mirrored-library-busybox                   1.34.1                       827365c7baf1   3 months ago    4.86MB
tccr.io/truecharts/storj-node                      <none>                       abd86310410e   3 months ago    124MB
<none>                                             <none>                       334e4a014c81   4 months ago    4.86MB
storjlabs/storagenode                              1d42f9ac3-v1.68.2-go1.18.8   5bd8d2096d18   4 months ago    124MB
quay.io/metallb/speaker                            v0.13.7                      738c5d221d60   5 months ago    106MB
quay.io/metallb/controller                         v0.13.7                      e73361dabfb8   5 months ago    58.7MB
quay.io/prometheus-operator/prometheus-operator    v0.60.1                      df968278084b   6 months ago    50.3MB
rancher/mirrored-coredns-coredns                   1.9.1                        99376d8f35e0   13 months ago   49.5MB
rancher/klipper-lb                                 v0.3.5                       dbd43b6716a0   14 months ago   8.09MB
openebs/zfs-driver                                 2.0.0                        2e17748d7c44   15 months ago   176MB
alpine/curl                                        latest                       6c746001fe6d   16 months ago   7.71MB
rancher/mirrored-pause                             3.6                          6270bb605e12   19 months ago   683kB
k8s.gcr.io/sig-storage/csi-provisioner             v3.0.0                       fe0f921f3c92   20 months ago   56.7MB
k8s.gcr.io/sig-storage/csi-node-driver-registrar   v2.3.0                       368ee7d6e60f   20 months ago   18.7MB
k8s.gcr.io/sig-storage/csi-resizer                 v1.2.0                       0aa9629e1508   23 months ago   54MB
k8s.gcr.io/sig-storage/csi-snapshotter             v4.0.0                       da32a49a903a   2 years ago     49.6MB
k8s.gcr.io/sig-storage/snapshot-controller         v4.0.0                       f1d8a00ae690   2 years ago     46.6MB
intel/intel-gpu-initcontainer                      0.19.0                       e026ae0f49c4   2 years ago     69.1MB
intel/intel-gpu-plugin                             0.19.0                       f2c60b24fad8   2 years ago     80MB
debug finished in 0 seconds for docker images -a


+--------------------------------------------------------------------------------+
+                     K3s Logs (last 1000 lines) @1681216843                     +
+--------------------------------------------------------------------------------+
-- Journal begins at Tue 2023-04-11 14:22:22 CEST, ends at Tue 2023-04-11 14:40:39 CEST. --
Apr 11 14:37:56 truenas systemd[1]: Starting Lightweight Kubernetes...
Apr 11 14:37:57 truenas k3s[6131]: time="2023-04-11T14:37:57+02:00" level=info msg="Starting k3s v1.25.3+k3s-9afcd6b9-dirty (9afcd6b9)"
Apr 11 14:37:57 truenas k3s[6131]: time="2023-04-11T14:37:57+02:00" level=info msg="Configuring sqlite3 database connection pooling: maxIdleConns=2, maxOpenConns=0, connMaxLifetime=0s"
Apr 11 14:37:57 truenas k3s[6131]: time="2023-04-11T14:37:57+02:00" level=info msg="Configuring database table schema and indexes, this may take a moment..."
Apr 11 14:37:58 truenas k3s[6131]: time="2023-04-11T14:37:58+02:00" level=info msg="Database tables and indexes are up to date"
Apr 11 14:37:58 truenas k3s[6131]: time="2023-04-11T14:37:58+02:00" level=info msg="Kine available at unix://kine.sock"
Apr 11 14:37:58 truenas k3s[6131]: time="2023-04-11T14:37:58+02:00" level=info msg="Reconciling bootstrap data between datastore and disk"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Running kube-apiserver --advertise-address=192.168.2.105 --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/log/k3s_server_audit.log --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/mnt/RaidZ1/ix-applications/k3s/server/tls/temporary-certs --client-ca-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt --egress-selector-config-file=/mnt/RaidZ1/ix-applications/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount --enable-aggregator-routing=true --etcd-servers=unix://kine.sock --feature-gates=MixedProtocolLBService=true --kubelet-certificate-authority=/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/service.key --service-cluster-ip-range=172.17.0.0/16 --service-node-port-range=9000-65535 --storage-backend=etcd3 --tls-cert-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/serving-kube-apiserver.crt --tls-private-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/serving-kube-apiserver.key"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Running kube-scheduler --authentication-kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/scheduler.kubeconfig --leader-elect=false --profiling=false --secure-port=10259"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cluster-cidr=172.16.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.key --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/controller.kubeconfig --leader-elect=false --node-cidr-mask-size=16 --profiling=false --root-ca-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/mnt/RaidZ1/ix-applications/k3s/server/tls/service.key --service-cluster-ip-range=172.17.0.0/16 --terminated-pod-gc-threshold=5 --use-service-account-credentials=true"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Running cloud-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/cloud-controller.kubeconfig --authorization-kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/cloud-controller.kubeconfig --bind-address=127.0.0.1 --cloud-config=/mnt/RaidZ1/ix-applications/k3s/server/etc/cloud-config.yaml --cloud-provider=k3s --cluster-cidr=172.16.0.0/16 --configure-cloud-routes=false --controllers=*,-route,-cloud-node,-cloud-node-lifecycle --kubeconfig=/mnt/RaidZ1/ix-applications/k3s/server/cred/cloud-controller.kubeconfig --leader-elect=false --leader-elect-resource-name=k3s-cloud-controller-manager --node-status-update-frequency=1m0s --profiling=false"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Tunnel server egress proxy mode: agent"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Tunnel server egress proxy waiting for runtime core to become available"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Waiting for API server to become available"
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.311703    6131 server.go:581] external host was not specified, using 192.168.2.105
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.339891    6131 server.go:171] Version: v1.25.3+k3s-9afcd6b9-dirty
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.340015    6131 server.go:173] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Server node token is available at /mnt/RaidZ1/ix-applications/k3s/server/token"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="To join server node to cluster: k3s server -s https://192.168.2.105:6443 -t ${SERVER_NODE_TOKEN}"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Agent node token is available at /mnt/RaidZ1/ix-applications/k3s/server/agent-token"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="To join agent node to cluster: k3s agent -s https://192.168.2.105:6443 -t ${AGENT_NODE_TOKEN}"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Wrote kubeconfig /etc/rancher/k3s/k3s.yaml"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Run: k3s kubectl"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="certificate CN=ix-truenas signed by CN=k3s-server-ca@1673294040: notBefore=2023-01-09 19:54:00 +0000 UTC notAfter=2024-04-10 12:37:59 +0000 UTC"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="certificate CN=system:node:ix-truenas,O=system:nodes signed by CN=k3s-client-ca@1673294040: notBefore=2023-01-09 19:54:00 +0000 UTC notAfter=2024-04-10 12:37:59 +0000 UTC"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Module overlay was already loaded"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Module br_netfilter was already loaded"
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.457840    6131 shared_informer.go:255] Waiting for caches to sync for node_authorizer
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.462017    6131 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.462100    6131 plugins.go:161] Loaded 11 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.465463    6131 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.465630    6131 plugins.go:161] Loaded 11 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
Apr 11 14:37:59 truenas k3s[6131]: {"level":"warn","ts":"2023-04-11T14:37:59.471+0200","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000761180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_max' to 131072"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_established' to 86400"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_close_wait' to 3600"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Running cri-dockerd --container-runtime-endpoint=unix:///run/k3s/cri-dockerd/cri-dockerd.sock --cri-dockerd-root-directory=/mnt/RaidZ1/ix-applications/k3s/agent/cri-dockerd --network-plugin=cni --pod-infra-container-image=rancher/mirrored-pause:3.6"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --client-ca-file=/mnt/RaidZ1/ix-applications/k3s/agent/client-ca.crt --cluster-dns=172.17.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/cri-dockerd/cri-dockerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ix-truenas --kubeconfig=/mnt/RaidZ1/ix-applications/k3s/agent/kubelet.kubeconfig --max-pods=250 --node-ip=0.0.0.0 --node-labels= --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/mnt/RaidZ1/ix-applications/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/etc/resolv.conf --serialize-image-pulls=false --tls-cert-file=/mnt/RaidZ1/ix-applications/k3s/agent/serving-kubelet.crt --tls-private-key-file=/mnt/RaidZ1/ix-applications/k3s/agent/serving-kubelet.key"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Connecting to docker on the Endpoint unix:///var/run/docker.sock"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Start docker client with request timeout 0s"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Connecting to proxy" url="wss://127.0.0.1:6443/v1-k3s/connect"
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.547931    6131 genericapiserver.go:656] Skipping API apiextensions.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.549449    6131 instance.go:261] Using reconciler: lease
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Handling backend connection request [ix-truenas]"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Hairpin mode is set to none"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Waiting to retrieve kube-proxy configuration; server is not ready: https://127.0.0.1:6443/v1-k3s/readyz: 500 Internal Server Error"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Loaded network plugin cni"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Docker cri networking managed by network plugin cni"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Docker Info: &{ID:EPOB:JRP7:WMCC:Z5CH:X3B7:P47Y:P4PY:SWEZ:46OV:BZW4:EEGS:SQ6O Containers:43 ContainersRunning:0 ContainersPaused:0 ContainersStopped:43 Images:25 Driver:overlay2 DriverStatus:[[Backing Filesystem zfs] [Supports d_type true] [Native Overlay Diff true] [userxattr false]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:false CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:22 OomKillDisable:false NGoroutines:41 SystemTime:2023-04-11T14:37:59.599237683+02:00 LoggingDriver:json-file CgroupDriver:cgroupfs CgroupVersion:2 NEventsListener:0 KernelVersion:5.15.79+truenas OperatingSystem:Debian GNU/Linux 11 (bullseye) OSVersion:11 OSType:linux Architecture:x86_64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:0xc0005219d0 NCPU:4 MemTotal:16283435008 GenericResources:[] DockerRootDir:/mnt/RaidZ1/ix-applications/docker HTTPProxy: HTTPSProxy: NoProxy: Name:truenas Labels:[] ExperimentalBuild:false ServerVersion:dev ClusterStore: ClusterAdvertise: Runtimes:map[io.containerd.runc.v2:{Path:runc Args:[] Shim:<nil>} io.containerd.runtime.v1.linux:{Path:runc Args:[] Shim:<nil>} runc:{Path:runc Args:[] Shim:<nil>}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:0 Cluster:<nil> Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:9cd3357b7fd7218e4aec3eae239db1f68a5a6ec6 Expected:9cd3357b7fd7218e4aec3eae239db1f68a5a6ec6} RuncCommit:{ID:v1.1.4-0-g5fd4c4d Expected:v1.1.4-0-g5fd4c4d} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=apparmor name=cgroupns] ProductLicense: DefaultAddressPools:[] Warnings:[]}"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Setting cgroupDriver cgroupfs"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Docker cri received runtime config &RuntimeConfig{NetworkConfig:&NetworkConfig{PodCidr:,},}"
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Starting the GRPC backend for the Docker CRI interface."
Apr 11 14:37:59 truenas k3s[6131]: time="2023-04-11T14:37:59+02:00" level=info msg="Start cri-dockerd grpc backend"
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.680771    6131 instance.go:574] API group "internal.apiserver.k8s.io" is not enabled, skipping.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.808793    6131 genericapiserver.go:656] Skipping API authentication.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.809876    6131 genericapiserver.go:656] Skipping API authorization.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.812909    6131 genericapiserver.go:656] Skipping API autoscaling/v2beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.815679    6131 genericapiserver.go:656] Skipping API batch/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.816830    6131 genericapiserver.go:656] Skipping API certificates.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.818048    6131 genericapiserver.go:656] Skipping API coordination.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.818088    6131 genericapiserver.go:656] Skipping API discovery.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.820715    6131 genericapiserver.go:656] Skipping API networking.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.820722    6131 genericapiserver.go:656] Skipping API networking.k8s.io/v1alpha1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.821575    6131 genericapiserver.go:656] Skipping API node.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.821582    6131 genericapiserver.go:656] Skipping API node.k8s.io/v1alpha1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.821621    6131 genericapiserver.go:656] Skipping API policy/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.824156    6131 genericapiserver.go:656] Skipping API rbac.authorization.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.824164    6131 genericapiserver.go:656] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.825028    6131 genericapiserver.go:656] Skipping API scheduling.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.825034    6131 genericapiserver.go:656] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.828197    6131 genericapiserver.go:656] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.830928    6131 genericapiserver.go:656] Skipping API flowcontrol.apiserver.k8s.io/v1alpha1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.833650    6131 genericapiserver.go:656] Skipping API apps/v1beta2 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.833658    6131 genericapiserver.go:656] Skipping API apps/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.834784    6131 genericapiserver.go:656] Skipping API admissionregistration.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.835848    6131 genericapiserver.go:656] Skipping API events.k8s.io/v1beta1 because it has no resources.
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.836412    6131 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Apr 11 14:37:59 truenas k3s[6131]: I0411 14:37:59.836424    6131 plugins.go:161] Loaded 11 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
Apr 11 14:37:59 truenas k3s[6131]: W0411 14:37:59.849126    6131 genericapiserver.go:656] Skipping API apiregistration.k8s.io/v1beta1 because it has no resources.
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.673820    6131 secure_serving.go:210] Serving securely on 127.0.0.1:6444
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.673945    6131 dynamic_serving_content.go:132] "Starting controller" name="serving-cert::/mnt/RaidZ1/ix-applications/k3s/server/tls/serving-kube-apiserver.crt::/mnt/RaidZ1/ix-applications/k3s/server/tls/serving-kube-apiserver.key"
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.673960    6131 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.674135    6131 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt"
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.674443    6131 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/mnt/RaidZ1/ix-applications/k3s/server/tls/request-header-ca.crt"
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.676219    6131 apiservice_controller.go:97] Starting APIServiceRegistrationController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.676246    6131 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.676368    6131 autoregister_controller.go:141] Starting autoregister controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.676373    6131 cache.go:32] Waiting for caches to sync for autoregister controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.676406    6131 controller.go:80] Starting OpenAPI V3 AggregationController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.678188    6131 available_controller.go:491] Starting AvailableConditionController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.678194    6131 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.678227    6131 controller.go:83] Starting OpenAPI AggregationController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.679093    6131 customresource_discovery_controller.go:209] Starting DiscoveryController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.679228    6131 dynamic_serving_content.go:132] "Starting controller" name="aggregator-proxy-cert::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-auth-proxy.crt::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-auth-proxy.key"
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.689559    6131 apf_controller.go:300] Starting API Priority and Fairness config controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.690226    6131 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.690233    6131 shared_informer.go:255] Waiting for caches to sync for cluster_authentication_trust_controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.690490    6131 crdregistration_controller.go:111] Starting crd-autoregister controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.690496    6131 shared_informer.go:255] Waiting for caches to sync for crd-autoregister
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.693504    6131 controller.go:85] Starting OpenAPI controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.693672    6131 controller.go:85] Starting OpenAPI V3 controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.693834    6131 naming_controller.go:291] Starting NamingConditionController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.693934    6131 establishing_controller.go:76] Starting EstablishingController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.694134    6131 nonstructuralschema_controller.go:192] Starting NonStructuralSchemaConditionController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.694250    6131 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.694278    6131 crd_finalizer.go:266] Starting CRDFinalizer
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.694436    6131 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt"
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.698695    6131 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/mnt/RaidZ1/ix-applications/k3s/server/tls/request-header-ca.crt"
Apr 11 14:38:00 truenas k3s[6131]: E0411 14:38:00.757694    6131 controller.go:163] Error removing old endpoints from kubernetes service: no master IPs were listed in storage, refusing to erase all endpoints for the kubernetes service
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.758497    6131 shared_informer.go:262] Caches are synced for node_authorizer
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.776357    6131 cache.go:39] Caches are synced for APIServiceRegistrationController controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.779376    6131 cache.go:39] Caches are synced for autoregister controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.779929    6131 cache.go:39] Caches are synced for AvailableConditionController controller
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.789780    6131 apf_controller.go:305] Running API Priority and Fairness config worker
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.790717    6131 shared_informer.go:262] Caches are synced for crd-autoregister
Apr 11 14:38:00 truenas k3s[6131]: I0411 14:38:00.791015    6131 shared_informer.go:262] Caches are synced for cluster_authentication_trust_controller
Apr 11 14:38:01 truenas k3s[6131]: I0411 14:38:01.534371    6131 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
Apr 11 14:38:01 truenas k3s[6131]: I0411 14:38:01.679673    6131 storage_scheduling.go:111] all system priority classes are created successfully or already exist.
Apr 11 14:38:02 truenas k3s[6131]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.556269    6131 server.go:199] "--pod-infra-container-image will not be pruned by the image garbage collector in kubelet and should also be set in the remote runtime"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Stopped tunnel to 127.0.0.1:6443"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Connecting to proxy" url="wss://192.168.2.105:6443/v1-k3s/connect"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Proxy done" err="context canceled" url="wss://127.0.0.1:6443/v1-k3s/connect"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="error in remotedialer server [400]: websocket: close 1006 (abnormal closure): unexpected EOF"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Tunnel authorizer set Kubelet Port 10250"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Handling backend connection request [ix-truenas]"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Annotations and labels have already set on node: ix-truenas"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.582181    6131 server.go:408] "Kubelet version" kubeletVersion="v1.25.3+k3s-9afcd6b9-dirty"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.582206    6131 server.go:410] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.583438    6131 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/RaidZ1/ix-applications/k3s/agent/client-ca.crt"
Apr 11 14:38:02 truenas k3s[6131]: W0411 14:38:02.584743    6131 manager.go:159] Cannot detect current cgroup on cgroup v2
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.606720    6131 server.go:655] "--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.607574    6131 container_manager_linux.go:262] "Container manager verified user specified cgroup-root exists" cgroupRoot=[]
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.607868    6131 container_manager_linux.go:267] "Creating Container Manager object based on Node Config" nodeConfig={RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: KubeletOOMScoreAdj:-999 ContainerRuntime: CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:true NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerPolicyOptions:map[] ExperimentalTopologyManagerScope:container ExperimentalCPUManagerReconcilePeriod:10s ExperimentalMemoryManagerPolicy:None ExperimentalMemoryManagerReservedMemory:[] ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.611966    6131 topology_manager.go:134] "Creating topology manager with policy per scope" topologyPolicyName="none" topologyScopeName="container"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.611984    6131 container_manager_linux.go:302] "Creating device plugin manager" devicePluginEnabled=true
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.612512    6131 state_mem.go:36] "Initialized new in-memory state store"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.622554    6131 kubelet.go:381] "Attempting to sync node with API server"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.622585    6131 kubelet.go:270] "Adding static pod path" path="/mnt/RaidZ1/ix-applications/k3s/agent/pod-manifests"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.623247    6131 kubelet.go:281] "Adding apiserver pod source"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.628396    6131 apiserver.go:42] "Waiting for node sync before watching apiserver pods"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.628650    6131 apiserver.go:52] "Watching apiserver"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.659313    6131 kuberuntime_manager.go:240] "Container runtime initialized" containerRuntime="docker" version="dev" apiVersion="1.41.0"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.662010    6131 server.go:1170] "Started kubelet"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.672133    6131 server.go:155] "Starting to listen" address="0.0.0.0" port=10250
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.672785    6131 scope.go:115] "RemoveContainer" containerID="666ad652f4ed1a2a135d850f7cf34fea2bb1e0d4e6cdf5d9992df69dfe089106"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.673502    6131 server.go:438] "Adding debug handlers to kubelet server"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.678500    6131 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.678948    6131 volume_manager.go:293] "Starting Kubelet Volume Manager"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.679159    6131 desired_state_of_world_populator.go:149] "Desired state populator starts to run"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.688210    6131 controller.go:616] quota admission added evaluator for: leases.coordination.k8s.io
Apr 11 14:38:02 truenas k3s[6131]: E0411 14:38:02.696823    6131 kubelet.go:2373] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Kube API server is now running"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="ETCD server is now running"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="k3s is up and running"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Waiting for cloud-controller-manager privileges to become available"
Apr 11 14:38:02 truenas systemd[1]: Started Lightweight Kubernetes.
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.761509    6131 cpu_manager.go:213] "Starting CPU manager" policy="none"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.761524    6131 cpu_manager.go:214] "Reconciling" reconcilePeriod="10s"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.761542    6131 state_mem.go:36] "Initialized new in-memory state store"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.787636    6131 kuberuntime_manager.go:1050] "Updating runtime config through cri with podcidr" CIDR="172.16.0.0/16"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.787658    6131 state_mem.go:88] "Updated default CPUSet" cpuSet=""
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.787780    6131 state_mem.go:96] "Updated CPUSet assignments" assignments=map[]
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.787810    6131 policy_none.go:49] "None policy: Start"
Apr 11 14:38:02 truenas k3s[6131]: time="2023-04-11T14:38:02+02:00" level=info msg="Docker cri received runtime config &RuntimeConfig{NetworkConfig:&NetworkConfig{PodCidr:172.16.0.0/16,},}"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.789270    6131 kubelet_network.go:60] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="172.16.0.0/16"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.889699    6131 kubelet_node_status.go:70] "Attempting to register node" node="ix-truenas"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.903896    6131 memory_manager.go:168] "Starting memorymanager" policy="None"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.904008    6131 state_mem.go:35] "Initializing new in-memory state store"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.904301    6131 state_mem.go:75] "Updated machine memory state"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.916999    6131 kubelet_node_status.go:108] "Node was previously registered" node="ix-truenas"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.928388    6131 scope.go:115] "RemoveContainer" containerID="9bb2a23fe8ef1ae0dc64241b8bf9b15cf4dc5edb36d2513c3e6a2454648ba45e"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.958720    6131 kubelet_node_status.go:73] "Successfully registered node" node="ix-truenas"
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.962914    6131 serving.go:355] Generated self-signed cert in-memory
Apr 11 14:38:02 truenas k3s[6131]: I0411 14:38:02.998806    6131 plugin_manager.go:118] "Starting Kubelet Plugin Manager"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.011428    6131 setters.go:545] "Node became not ready" node="ix-truenas" condition={Type:Ready Status:False LastHeartbeatTime:2023-04-11 14:38:03.01138311 +0200 CEST m=+5.729298110 LastTransitionTime:2023-04-11 14:38:03.01138311 +0200 CEST m=+5.729298110 Reason:KubeletNotReady Message:[PLEG is not healthy: pleg has yet to be successful, container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized]}
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.051078    6131 serving.go:355] Generated self-signed cert in-memory
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.058914    6131 kubelet.go:2373] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.128559    6131 scope.go:115] "RemoveContainer" containerID="f55c5da82ffb7e17d114265a2d196988bf401cab31785f59dd1dc9bcba35c825"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.320404    6131 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv4
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Applying CRD addons.k3s.cattle.io"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.422842    6131 controllermanager.go:178] Version: v1.25.3+k3s-9afcd6b9-dirty
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.422943    6131 controllermanager.go:180] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.426906    6131 secure_serving.go:210] Serving securely on 127.0.0.1:10257
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427570    6131 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427624    6131 shared_informer.go:255] Waiting for caches to sync for RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427673    6131 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427734    6131 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427776    6131 shared_informer.go:255] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427823    6131 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.427865    6131 shared_informer.go:255] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.448287    6131 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv6
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.448813    6131 status_manager.go:161] "Starting to sync pod status with apiserver"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.449033    6131 kubelet.go:2010] "Starting kubelet main sync loop"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.449088    6131 kubelet.go:2034] "Skipping pod synchronization" err="PLEG is not healthy: pleg has yet to be successful"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Applying CRD helmcharts.helm.cattle.io"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.485768    6131 scope.go:115] "RemoveContainer" containerID="209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.494569    6131 server.go:148] "Starting Kubernetes Scheduler" version="v1.25.3+k3s-9afcd6b9-dirty"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.494688    6131 server.go:150] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.498330    6131 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.498410    6131 shared_informer.go:255] Waiting for caches to sync for RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.498462    6131 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.498506    6131 shared_informer.go:255] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.498553    6131 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.498616    6131 shared_informer.go:255] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.502222    6131 secure_serving.go:210] Serving securely on 127.0.0.1:10259
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.502390    6131 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"netdata-7bc8fbb7b4-8vz7s_ix-netdata\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"083141b01aa9278834ec5349e0cc12f54c6731fabded3ebfe8aa4c7d50a13c50\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"netdata-7bc8fbb7b4-8vz7s_ix-netdata\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"2161fcd66b50229d0f9ef8ab9d58701f9b2642d641b8b2216d8dd36379a458a1\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"afb1c58515188387bb0d2173c55eedb8d180038d745a4d81fb12e53107648cf3\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"5c06fe7fa02615a5d1256ea32d65ad003d99778cf26d7e00dfe8bd7cef928151\""
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.528490    6131 shared_informer.go:262] Caches are synced for RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.528498    6131 shared_informer.go:262] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.528490    6131 shared_informer.go:262] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"coredns-75fc8f8fff-dnkbk_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"23508512b7c2cc3342b3135ebfbf5441a2b35a4b65a78ab89db9ae5d72be73ed\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"coredns-75fc8f8fff-dnkbk_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"21016ba7a711cca48be36b1bb82c6eff7e206bc019c5d4a035c8855fdb55fdae\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"controller-84d6d4db45-2jc2g_metallb-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"242211d179798ed017f0c5ef4fa41f9219b7819ab04ead8cd70571edfb6ec6f2\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"controller-84d6d4db45-2jc2g_metallb-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"4082d02e09502b0f146c69afae78946220a91b86e6d920cd54693c8e9d3713ba\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"aad36eaac89b80ce5df6ed0f54bc7163108ea42b19df455dfbd4ba1de2f3a144\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"dbfc0b44cb38fe556e4fc4fdc53702c0ea5c56e755484332294d2c556ddf79d2\""
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.550031    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.550723    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.550756    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.550939    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.550985    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551017    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551048    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551077    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551107    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551141    6131 topology_manager.go:205] "Topology Admit Handler"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551183    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="25a463805bb0221809e90f7b83b278f7929c67f6c783dca08ff2894d066ac058"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551193    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f58938bcbf92d7f8ec5e87a428ac1598ead589af57e46d88e9dbeabb9c093f4f"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551200    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f55c5da82ffb7e17d114265a2d196988bf401cab31785f59dd1dc9bcba35c825"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551206    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="cde74b38e823cfbfad10f7d140eb388fc0a48c648e81c270255b71f0215aea90"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551212    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="95347006c898783ec837321ad39f787bacae0da13dc0b531aa689d09a98a2e58"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551220    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="083141b01aa9278834ec5349e0cc12f54c6731fabded3ebfe8aa4c7d50a13c50"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551226    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="2161fcd66b50229d0f9ef8ab9d58701f9b2642d641b8b2216d8dd36379a458a1"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551233    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="afb1c58515188387bb0d2173c55eedb8d180038d745a4d81fb12e53107648cf3"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551239    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="5c06fe7fa02615a5d1256ea32d65ad003d99778cf26d7e00dfe8bd7cef928151"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551247    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="23508512b7c2cc3342b3135ebfbf5441a2b35a4b65a78ab89db9ae5d72be73ed"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551252    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="21016ba7a711cca48be36b1bb82c6eff7e206bc019c5d4a035c8855fdb55fdae"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551262    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="242211d179798ed017f0c5ef4fa41f9219b7819ab04ead8cd70571edfb6ec6f2"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.551267    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="4082d02e09502b0f146c69afae78946220a91b86e6d920cd54693c8e9d3713ba"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.564198    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s" podUID=c4438af0-7353-463f-9a23-1d1bd583a23d
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.564407    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="metallb-system/controller-84d6d4db45-2jc2g" podUID=24978195-ae80-42c9-80e3-6e880be2bac2
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.564475    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-75fc8f8fff-dnkbk" podUID=30a4863f-b5c5-44ac-bd90-df544fd65f1b
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.564519    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq" podUID=c0561815-b5dd-4748-b1d5-fb362bcd5682
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.569164    6131 scope.go:115] "RemoveContainer" containerID="e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.575210    6131 serving.go:355] Generated self-signed cert in-memory
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.584414    6131 scope.go:115] "RemoveContainer" containerID="8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Applying CRD helmchartconfigs.helm.cattle.io"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.598518    6131 shared_informer.go:262] Caches are synced for RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.598601    6131 shared_informer.go:262] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.598668    6131 shared_informer.go:262] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.638145    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="aad36eaac89b80ce5df6ed0f54bc7163108ea42b19df455dfbd4ba1de2f3a144"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.638160    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="dbfc0b44cb38fe556e4fc4fdc53702c0ea5c56e755484332294d2c556ddf79d2"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.638448    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=d03d171f-7cf6-4f50-878b-5cb4e9023b3e
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"intel-gpu-plugin-zvqhc_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"f6851aee517b1b8ac03036b3f5cbfbbbaa550b4b3a48d05c3d1a402c15fccbea\""
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"intel-gpu-plugin-zvqhc_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"489d3f546c31415786266c5343f3de1bdd4bed99f204bfeec1c59b570ba13ec6\""
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.649242    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f6851aee517b1b8ac03036b3f5cbfbbbaa550b4b3a48d05c3d1a402c15fccbea"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.652692    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="489d3f546c31415786266c5343f3de1bdd4bed99f204bfeec1c59b570ba13ec6"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.650714    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/intel-gpu-plugin-zvqhc" podUID=ee2f204d-7d95-4555-b933-b6ba87ba1547
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.657405    6131 scope.go:115] "RemoveContainer" containerID="b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.674794    6131 scope.go:115] "RemoveContainer" containerID="26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696449    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"netdatalib\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-netdatalib\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696564    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"sysfs\" (UniqueName: \"kubernetes.io/host-path/ee2f204d-7d95-4555-b933-b6ba87ba1547-sysfs\") pod \"intel-gpu-plugin-zvqhc\" (UID: \"ee2f204d-7d95-4555-b933-b6ba87ba1547\") " pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696609    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-7f85s\" (UniqueName: \"kubernetes.io/projected/ee2f204d-7d95-4555-b933-b6ba87ba1547-kube-api-access-7f85s\") pod \"intel-gpu-plugin-zvqhc\" (UID: \"ee2f204d-7d95-4555-b933-b6ba87ba1547\") " pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696634    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-574qt\" (UniqueName: \"kubernetes.io/projected/30a4863f-b5c5-44ac-bd90-df544fd65f1b-kube-api-access-574qt\") pod \"coredns-75fc8f8fff-dnkbk\" (UID: \"30a4863f-b5c5-44ac-bd90-df544fd65f1b\") " pod="kube-system/coredns-75fc8f8fff-dnkbk"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696652    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-encr-keys\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696665    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-host-root\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696686    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-registration-dir\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696701    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-pods-mount-dir\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696713    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"os-release\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-os-release\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696734    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-gd6g7\" (UniqueName: \"kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696751    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume\") pod \"coredns-75fc8f8fff-dnkbk\" (UID: \"30a4863f-b5c5-44ac-bd90-df544fd65f1b\") " pod="kube-system/coredns-75fc8f8fff-dnkbk"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696766    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/d03d171f-7cf6-4f50-878b-5cb4e9023b3e-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"d03d171f-7cf6-4f50-878b-5cb4e9023b3e\") " pod="kube-system/openebs-zfs-controller-0"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696778    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kubeletsockets\" (UniqueName: \"kubernetes.io/host-path/ee2f204d-7d95-4555-b933-b6ba87ba1547-kubeletsockets\") pod \"intel-gpu-plugin-zvqhc\" (UID: \"ee2f204d-7d95-4555-b933-b6ba87ba1547\") " pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696792    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-sphlt\" (UniqueName: \"kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt\") pod \"prometheus-operator-5dcffb7cb8-q6xxq\" (UID: \"c0561815-b5dd-4748-b1d5-fb362bcd5682\") " pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696808    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-nqtx7\" (UniqueName: \"kubernetes.io/projected/d03d171f-7cf6-4f50-878b-5cb4e9023b3e-kube-api-access-nqtx7\") pod \"openebs-zfs-controller-0\" (UID: \"d03d171f-7cf6-4f50-878b-5cb4e9023b3e\") " pod="kube-system/openebs-zfs-controller-0"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696831    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"netdataconfig\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-netdataconfig\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696844    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"proc\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-proc\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696856    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"sys\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-sys\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696868    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nfd-source-hooks\" (UniqueName: \"kubernetes.io/host-path/ee2f204d-7d95-4555-b933-b6ba87ba1547-nfd-source-hooks\") pod \"intel-gpu-plugin-zvqhc\" (UID: \"ee2f204d-7d95-4555-b933-b6ba87ba1547\") " pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696881    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-8k2r4\" (UniqueName: \"kubernetes.io/projected/24978195-ae80-42c9-80e3-6e880be2bac2-kube-api-access-8k2r4\") pod \"controller-84d6d4db45-2jc2g\" (UID: \"24978195-ae80-42c9-80e3-6e880be2bac2\") " pod="metallb-system/controller-84d6d4db45-2jc2g"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696895    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-device-dir\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696909    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"custom-config-volume\" (UniqueName: \"kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume\") pod \"coredns-75fc8f8fff-dnkbk\" (UID: \"30a4863f-b5c5-44ac-bd90-df544fd65f1b\") " pod="kube-system/coredns-75fc8f8fff-dnkbk"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696921    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-chroot-zfs\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696935    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-9tqfl\" (UniqueName: \"kubernetes.io/projected/c1e17754-0080-4f9e-9fc0-4a63b0e92e6e-kube-api-access-9tqfl\") pod \"speaker-rcq54\" (UID: \"c1e17754-0080-4f9e-9fc0-4a63b0e92e6e\") " pod="metallb-system/speaker-rcq54"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696948    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"user\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-user\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.696961    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"group\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-group\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.697002    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cert\" (UniqueName: \"kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert\") pod \"controller-84d6d4db45-2jc2g\" (UID: \"24978195-ae80-42c9-80e3-6e880be2bac2\") " pod="metallb-system/controller-84d6d4db45-2jc2g"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.697015    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-plugin-dir\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.697028    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"netdatacache\" (UniqueName: \"kubernetes.io/host-path/c4438af0-7353-463f-9a23-1d1bd583a23d-netdatacache\") pod \"netdata-7bc8fbb7b4-8vz7s\" (UID: \"c4438af0-7353-463f-9a23-1d1bd583a23d\") " pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.697041    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"devfs\" (UniqueName: \"kubernetes.io/host-path/ee2f204d-7d95-4555-b933-b6ba87ba1547-devfs\") pod \"intel-gpu-plugin-zvqhc\" (UID: \"ee2f204d-7d95-4555-b933-b6ba87ba1547\") " pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.697053    6131 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-zqmlw\" (UniqueName: \"kubernetes.io/projected/2a0727c6-327f-47a2-8bb9-c8741fd9f6ed-kube-api-access-zqmlw\") pod \"openebs-zfs-node-bwxzr\" (UID: \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\") " pod="kube-system/openebs-zfs-node-bwxzr"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.697062    6131 reconciler.go:169] "Reconciler: start to sync state"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.710006    6131 scope.go:115] "RemoveContainer" containerID="e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.710856    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163" containerID="e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.710874    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163} err="failed to get container status \"e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163\": rpc error: code = Unknown desc = Error: No such container: e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.710884    6131 scope.go:115] "RemoveContainer" containerID="b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.716848    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640" containerID="b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.716870    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640} err="failed to get container status \"b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640\": rpc error: code = Unknown desc = Error: No such container: b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.723053    6131 scope.go:115] "RemoveContainer" containerID="209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.723488    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d" containerID="209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.723555    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d} err="failed to get container status \"209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d\": rpc error: code = Unknown desc = Error: No such container: 209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.723604    6131 scope.go:115] "RemoveContainer" containerID="209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.723900    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d} err="failed to get container status \"209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d\": rpc error: code = Unknown desc = Error: No such container: 209a79bd0a6e766d7c35083d4f7ddf511edc92cf9638faa10ff0246eb1447c2d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.723950    6131 scope.go:115] "RemoveContainer" containerID="c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.767840    6131 scope.go:115] "RemoveContainer" containerID="e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.786481    6131 controllermanager.go:145] Version: v1.25.3+k3s-9afcd6b9-dirty
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788201    6131 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788212    6131 shared_informer.go:255] Waiting for caches to sync for RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788224    6131 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788230    6131 shared_informer.go:255] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788238    6131 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788241    6131 shared_informer.go:255] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788477    6131 secure_serving.go:210] Serving securely on 127.0.0.1:10258
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.788542    6131 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.801240    6131 configmap.go:197] Couldn't get configMap kube-system/coredns-custom: object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.802003    6131 secret.go:192] Couldn't get secret metallb-system/webhook-server-cert: object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.802112    6131 configmap.go:197] Couldn't get configMap kube-system/coredns: object "kube-system"/"coredns" not registered
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.815320    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:04.301277208 +0200 CEST m=+7.019192207 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "custom-config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.815363    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert podName:24978195-ae80-42c9-80e3-6e880be2bac2 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:04.315346169 +0200 CEST m=+7.033261173 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "cert" (UniqueName: "kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert") pod "controller-84d6d4db45-2jc2g" (UID: "24978195-ae80-42c9-80e3-6e880be2bac2") : object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.815376    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:04.315370062 +0200 CEST m=+7.033285062 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns" not registered
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.815476    6131 scope.go:115] "RemoveContainer" containerID="c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.815785    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030" containerID="c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.815799    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030} err="failed to get container status \"c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030\": rpc error: code = Unknown desc = Error: No such container: c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.815806    6131 scope.go:115] "RemoveContainer" containerID="3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.830506    6131 scope.go:115] "RemoveContainer" containerID="e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.830908    6131 kuberuntime_gc.go:146] "Failed to remove container" err="failed to get container status \"e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163\": rpc error: code = Unknown desc = Error: No such container: e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163" containerID="e7e6ef9a6c57edc27919a49b3b5eac7451c089a6000932d041570045077be163"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.830918    6131 scope.go:115] "RemoveContainer" containerID="3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.840484    6131 scope.go:115] "RemoveContainer" containerID="3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.841513    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75" containerID="3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.841535    6131 kuberuntime_gc.go:146] "Failed to remove container" err="failed to get container status \"3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75\": rpc error: code = Unknown desc = Error: No such container: 3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75" containerID="3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.841544    6131 scope.go:115] "RemoveContainer" containerID="c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.841659    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75} err="failed to get container status \"3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75\": rpc error: code = Unknown desc = Error: No such container: 3cec806cd204065d5f59bfdd3ef33ec9f1b278aeec6e89814bd67af2c94f5c75"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.841708    6131 scope.go:115] "RemoveContainer" containerID="b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.842176    6131 kuberuntime_gc.go:146] "Failed to remove container" err="failed to get container status \"c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030\": rpc error: code = Unknown desc = Error: No such container: c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030" containerID="c604c34c6f469ac91953735d1469fc96cc760d1249e98445706c5a83b706b030"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.842185    6131 scope.go:115] "RemoveContainer" containerID="b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing static file: /mnt/RaidZ1/ix-applications/k3s/server/static/charts/traefik-crd-10.19.300.tgz"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing static file: /mnt/RaidZ1/ix-applications/k3s/server/static/charts/traefik-10.19.300.tgz"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing manifest: /mnt/RaidZ1/ix-applications/k3s/server/manifests/kuberouter.yaml"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing manifest: /mnt/RaidZ1/ix-applications/k3s/server/manifests/multus-daemonset.yaml"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing manifest: /mnt/RaidZ1/ix-applications/k3s/server/manifests/zfs-operator.yaml"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing manifest: /mnt/RaidZ1/ix-applications/k3s/server/manifests/ccm.yaml"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing manifest: /mnt/RaidZ1/ix-applications/k3s/server/manifests/coredns.yaml"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Writing manifest: /mnt/RaidZ1/ix-applications/k3s/server/manifests/rolebindings.yaml"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.865129    6131 remote_runtime.go:531] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74\": Error response from daemon: removal of container b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74 is already in progress" containerID="b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.865166    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74} err="rpc error: code = Unknown desc = failed to remove container \"b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74\": Error response from daemon: removal of container b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74 is already in progress"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.865175    6131 scope.go:115] "RemoveContainer" containerID="26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.865847    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8" containerID="26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.865877    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8} err="failed to get container status \"26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8\": rpc error: code = Unknown desc = Error: No such container: 26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.865883    6131 scope.go:115] "RemoveContainer" containerID="e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.866084    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992" containerID="e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.866099    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992} err="failed to get container status \"e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992\": rpc error: code = Unknown desc = Error: No such container: e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.866106    6131 scope.go:115] "RemoveContainer" containerID="8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.866298    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad" containerID="8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.866351    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad} err="failed to get container status \"8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad\": rpc error: code = Unknown desc = Error: No such container: 8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.866400    6131 scope.go:115] "RemoveContainer" containerID="b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.868610    6131 remote_runtime.go:531] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74\": Error response from daemon: removal of container b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74 is already in progress" containerID="b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.868685    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74} err="rpc error: code = Unknown desc = failed to remove container \"b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74\": Error response from daemon: removal of container b12e4e92d019e9cbb4b29f974da8b0907731cc029b3110a418e5d50d60c2dd74 is already in progress"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.868734    6131 scope.go:115] "RemoveContainer" containerID="26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.868935    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8} err="failed to get container status \"26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8\": rpc error: code = Unknown desc = Error: No such container: 26832b459c2c75fece54a423094a62d9abf7c2050b5c5ca2d62b7cebedba68d8"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.868983    6131 scope.go:115] "RemoveContainer" containerID="e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.869214    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992} err="failed to get container status \"e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992\": rpc error: code = Unknown desc = Error: No such container: e6dd82ee79520764fcd674bddb30d27ced27c279f3523536a67a498a9287e992"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.869262    6131 scope.go:115] "RemoveContainer" containerID="8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.869452    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad} err="failed to get container status \"8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad\": rpc error: code = Unknown desc = Error: No such container: 8c1e55ce8de022d55ca4451068c72829f589f080b5ea04e41b271065c66d6cad"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.869499    6131 scope.go:115] "RemoveContainer" containerID="405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.881366    6131 scope.go:115] "RemoveContainer" containerID="405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.888277    6131 shared_informer.go:262] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.888298    6131 shared_informer.go:262] Caches are synced for RequestHeaderAuthRequestController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.888331    6131 shared_informer.go:262] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.889533    6131 remote_runtime.go:531] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d\": Error response from daemon: removal of container 405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d is already in progress" containerID="405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.889564    6131 kuberuntime_gc.go:146] "Failed to remove container" err="rpc error: code = Unknown desc = failed to remove container \"405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d\": Error response from daemon: removal of container 405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d is already in progress" containerID="405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.889574    6131 scope.go:115] "RemoveContainer" containerID="b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.889808    6131 kuberuntime_gc.go:146] "Failed to remove container" err="failed to get container status \"b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640\": rpc error: code = Unknown desc = Error: No such container: b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640" containerID="b0994539f5348bd7c087566ac580749c75d2bebe6dcd238dbd8acff40aca4640"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.893358    6131 scope.go:115] "RemoveContainer" containerID="405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: E0411 14:38:03.893746    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d" containerID="405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.893776    6131 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d} err="failed to get container status \"405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d\": rpc error: code = Unknown desc = Error: No such container: 405c3716ddc9785ddbdf4726748357c06f537c4223407e3c1d8c35e80d44952d"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.936840    6131 shared_informer.go:255] Waiting for caches to sync for tokens
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.938983    6131 controllermanager.go:603] Started "endpointslicemirroring"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.939162    6131 endpointslicemirroring_controller.go:216] Starting EndpointSliceMirroring controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.939172    6131 shared_informer.go:255] Waiting for caches to sync for endpoint_slice_mirroring
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.940679    6131 controllermanager.go:603] Started "podgc"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.940851    6131 gc_controller.go:99] Starting GC controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.940858    6131 shared_informer.go:255] Waiting for caches to sync for GC
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.945249    6131 controllermanager.go:603] Started "horizontalpodautoscaling"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947128    6131 horizontal.go:168] Starting HPA controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947141    6131 shared_informer.go:255] Waiting for caches to sync for HPA
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947325    6131 node_lifecycle_controller.go:497] Controller will reconcile labels.
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947375    6131 controllermanager.go:603] Started "nodelifecycle"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947470    6131 node_lifecycle_controller.go:532] Sending events to api server.
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947483    6131 node_lifecycle_controller.go:543] Starting node controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.947487    6131 shared_informer.go:255] Waiting for caches to sync for taint
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.948606    6131 controllermanager.go:603] Started "pv-protection"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.948787    6131 pv_protection_controller.go:79] Starting PV protection controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.948793    6131 shared_informer.go:255] Waiting for caches to sync for PV protection
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.949937    6131 controllermanager.go:603] Started "replicationcontroller"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.950189    6131 replica_set.go:205] Starting replicationcontroller controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.950196    6131 shared_informer.go:255] Waiting for caches to sync for ReplicationController
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.951448    6131 controllermanager.go:603] Started "csrapproving"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.951527    6131 certificate_controller.go:112] Starting certificate controller "csrapproving"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.951536    6131 shared_informer.go:255] Waiting for caches to sync for certificate-csrapproving
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.975069    6131 controllermanager.go:603] Started "persistentvolume-binder"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.975185    6131 pv_controller_base.go:318] Starting persistent volume controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.975193    6131 shared_informer.go:255] Waiting for caches to sync for persistent volume
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Starting k3s.cattle.io/v1, Kind=Addon controller"
Apr 11 14:38:03 truenas k3s[6131]: time="2023-04-11T14:38:03+02:00" level=info msg="Creating deploy event broadcaster"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.978297    6131 event.go:294] "Event occurred" object="kube-system/ccm" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/ccm.yaml\""
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.998307    6131 controllermanager.go:603] Started "persistentvolume-expander"
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.998501    6131 expand_controller.go:340] Starting expand controller
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.998549    6131 shared_informer.go:255] Waiting for caches to sync for expand
Apr 11 14:38:03 truenas k3s[6131]: I0411 14:38:03.999968    6131 controllermanager.go:603] Started "ttl-after-finished"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.000041    6131 ttlafterfinished_controller.go:109] Starting TTL after finished controller
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.000048    6131 shared_informer.go:255] Waiting for caches to sync for TTL after finished
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.001194    6131 controllermanager.go:603] Started "ephemeral-volume"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.001381    6131 controller.go:169] Starting ephemeral volume controller
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.001389    6131 shared_informer.go:255] Waiting for caches to sync for ephemeral
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.003153    6131 controllermanager.go:603] Started "cronjob"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.003347    6131 cronjob_controllerv2.go:135] "Starting cronjob controller v2"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.003357    6131 shared_informer.go:255] Waiting for caches to sync for cronjob
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Cluster dns configmap already exists"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Labels and annotations have been set successfully on node: ix-truenas"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009289    6131 controllermanager.go:603] Started "csrsigning"
Apr 11 14:38:04 truenas k3s[6131]: W0411 14:38:04.009301    6131 controllermanager.go:568] "tokencleaner" is disabled
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009373    6131 certificate_controller.go:112] Starting certificate controller "csrsigning-kubelet-serving"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009380    6131 shared_informer.go:255] Waiting for caches to sync for certificate-csrsigning-kubelet-serving
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009390    6131 certificate_controller.go:112] Starting certificate controller "csrsigning-kubelet-client"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009393    6131 shared_informer.go:255] Waiting for caches to sync for certificate-csrsigning-kubelet-client
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009401    6131 certificate_controller.go:112] Starting certificate controller "csrsigning-kube-apiserver-client"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009405    6131 shared_informer.go:255] Waiting for caches to sync for certificate-csrsigning-kube-apiserver-client
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009414    6131 certificate_controller.go:112] Starting certificate controller "csrsigning-legacy-unknown"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009417    6131 shared_informer.go:255] Waiting for caches to sync for certificate-csrsigning-legacy-unknown
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.009426    6131 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.crt::/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.key"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.010821    6131 node_ipam_controller.go:91] Sending events to api server.
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.010891    6131 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.key"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.012485    6131 controller.go:616] quota admission added evaluator for: rolebindings.rbac.authorization.k8s.io
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.013512    6131 event.go:294] "Event occurred" object="kube-system/ccm" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/ccm.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.014752    6131 controller.go:616] quota admission added evaluator for: addons.k3s.cattle.io
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.017683    6131 event.go:294] "Event occurred" object="kube-system/coredns" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/coredns.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.018157    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"netdata-7bc8fbb7b4-8vz7s_ix-netdata\" network: cni config uninitialized" podSandboxID="2161fcd66b50229d0f9ef8ab9d58701f9b2642d641b8b2216d8dd36379a458a1"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.018175    6131 kuberuntime_gc.go:176] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"netdata-7bc8fbb7b4-8vz7s_ix-netdata\" network: cni config uninitialized" sandboxID="2161fcd66b50229d0f9ef8ab9d58701f9b2642d641b8b2216d8dd36379a458a1"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.018795    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator\" network: cni config uninitialized" podSandboxID="5c06fe7fa02615a5d1256ea32d65ad003d99778cf26d7e00dfe8bd7cef928151"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.018809    6131 kuberuntime_gc.go:176] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"prometheus-operator-5dcffb7cb8-q6xxq_prometheus-operator\" network: cni config uninitialized" sandboxID="5c06fe7fa02615a5d1256ea32d65ad003d99778cf26d7e00dfe8bd7cef928151"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.019070    6131 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.crt::/mnt/RaidZ1/ix-applications/k3s/server/tls/client-ca.key"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.021668    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"coredns-75fc8f8fff-dnkbk_kube-system\" network: cni config uninitialized" podSandboxID="21016ba7a711cca48be36b1bb82c6eff7e206bc019c5d4a035c8855fdb55fdae"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.021682    6131 kuberuntime_gc.go:176] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"coredns-75fc8f8fff-dnkbk_kube-system\" network: cni config uninitialized" sandboxID="21016ba7a711cca48be36b1bb82c6eff7e206bc019c5d4a035c8855fdb55fdae"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.023992    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"controller-84d6d4db45-2jc2g_metallb-system\" network: cni config uninitialized" podSandboxID="4082d02e09502b0f146c69afae78946220a91b86e6d920cd54693c8e9d3713ba"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.024006    6131 kuberuntime_gc.go:176] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"controller-84d6d4db45-2jc2g_metallb-system\" network: cni config uninitialized" sandboxID="4082d02e09502b0f146c69afae78946220a91b86e6d920cd54693c8e9d3713ba"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.024255    6131 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.crt::/mnt/RaidZ1/ix-applications/k3s/server/tls/server-ca.key"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.024878    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"openebs-zfs-controller-0_kube-system\" network: cni config uninitialized" podSandboxID="dbfc0b44cb38fe556e4fc4fdc53702c0ea5c56e755484332294d2c556ddf79d2"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.024893    6131 kuberuntime_gc.go:176] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"openebs-zfs-controller-0_kube-system\" network: cni config uninitialized" sandboxID="dbfc0b44cb38fe556e4fc4fdc53702c0ea5c56e755484332294d2c556ddf79d2"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.037450    6131 shared_informer.go:262] Caches are synced for tokens
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.060494    6131 controller.go:616] quota admission added evaluator for: deployments.apps
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.064963    6131 event.go:294] "Event occurred" object="kube-system/coredns" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/coredns.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.068394    6131 event.go:294] "Event occurred" object="kube-system/kuberouter" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/kuberouter.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.075587    6131 event.go:294] "Event occurred" object="kube-system/kuberouter" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/kuberouter.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.078619    6131 event.go:294] "Event occurred" object="kube-system/multus-daemonset" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/multus-daemonset.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChartConfig controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChart controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting batch/v1, Kind=Job controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting /v1, Kind=Node controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting /v1, Kind=ConfigMap controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting /v1, Kind=ServiceAccount controller"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.147836    6131 event.go:294] "Event occurred" object="kube-system/multus-daemonset" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/multus-daemonset.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.150221    6131 event.go:294] "Event occurred" object="kube-system/rolebindings" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/rolebindings.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.168003    6131 event.go:294] "Event occurred" object="kube-system/rolebindings" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/rolebindings.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.193165    6131 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Creating service-controller event broadcaster"
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.296645    6131 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/RaidZ1/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.297007    6131 controller.go:616] quota admission added evaluator for: namespaces
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.301082    6131 controller.go:616] quota admission added evaluator for: serviceaccounts
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.306862    6131 configmap.go:197] Couldn't get configMap kube-system/coredns-custom: object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.306895    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:05.306886982 +0200 CEST m=+8.024801982 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "custom-config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.407153    6131 configmap.go:197] Couldn't get configMap kube-system/coredns: object "kube-system"/"coredns" not registered
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.407184    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:05.407175754 +0200 CEST m=+8.125090754 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns" not registered
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.408957    6131 secret.go:192] Couldn't get secret metallb-system/webhook-server-cert: object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.408985    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert podName:24978195-ae80-42c9-80e3-6e880be2bac2 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:05.408977387 +0200 CEST m=+8.126892387 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "cert" (UniqueName: "kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert") pod "controller-84d6d4db45-2jc2g" (UID: "24978195-ae80-42c9-80e3-6e880be2bac2") : object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting /v1, Kind=Secret controller"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Updating TLS secret for kube-system/k3s-serving (count: 13): map[listener.cattle.io/cn-0.0.0.0:0.0.0.0 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/cn-172.17.0.1:172.17.0.1 listener.cattle.io/cn-192.168.2.105:192.168.2.105 listener.cattle.io/cn-192.168.3.13:192.168.3.13 listener.cattle.io/cn-192.168.3.16:192.168.3.16 listener.cattle.io/cn-__1-f16284:::1 listener.cattle.io/cn-ix-truenas:ix-truenas listener.cattle.io/cn-kubernetes:kubernetes listener.cattle.io/cn-kubernetes.default:kubernetes.default listener.cattle.io/cn-kubernetes.default.svc:kubernetes.default.svc listener.cattle.io/cn-kubernetes.default.svc.cluster.local:kubernetes.default.svc.cluster.local listener.cattle.io/cn-localhost:localhost listener.cattle.io/fingerprint:SHA1=675282FB2710A15982F19457131988E568F67CDF]"
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Both sandbox container and checkpoint could not be found with id \"f58938bcbf92d7f8ec5e87a428ac1598ead589af57e46d88e9dbeabb9c093f4f\". Proceed without further sandbox information."
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Both sandbox container and checkpoint could not be found with id \"95347006c898783ec837321ad39f787bacae0da13dc0b531aa689d09a98a2e58\". Proceed without further sandbox information."
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490335    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"_\" network: cni config uninitialized" podSandboxID="f58938bcbf92d7f8ec5e87a428ac1598ead589af57e46d88e9dbeabb9c093f4f"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490355    6131 kuberuntime_manager.go:954] "Failed to stop sandbox" podSandboxID={Type:docker ID:f58938bcbf92d7f8ec5e87a428ac1598ead589af57e46d88e9dbeabb9c093f4f}
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490427    6131 remote_runtime.go:269] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"_\" network: cni config uninitialized" podSandboxID="95347006c898783ec837321ad39f787bacae0da13dc0b531aa689d09a98a2e58"
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490477    6131 kuberuntime_manager.go:954] "Failed to stop sandbox" podSandboxID={Type:docker ID:95347006c898783ec837321ad39f787bacae0da13dc0b531aa689d09a98a2e58}
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490534    6131 kuberuntime_manager.go:695] "killPodWithSyncResult failed" err="failed to \"KillPodSandbox\" for \"c1e17754-0080-4f9e-9fc0-4a63b0e92e6e\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"_\\\" network: cni config uninitialized\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490599    6131 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"c1e17754-0080-4f9e-9fc0-4a63b0e92e6e\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"_\\\" network: cni config uninitialized\"" pod="metallb-system/speaker-rcq54" podUID=c1e17754-0080-4f9e-9fc0-4a63b0e92e6e
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490499    6131 kuberuntime_manager.go:695] "killPodWithSyncResult failed" err="failed to \"KillPodSandbox\" for \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"_\\\" network: cni config uninitialized\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.490664    6131 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"2a0727c6-327f-47a2-8bb9-c8741fd9f6ed\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"_\\\" network: cni config uninitialized\"" pod="kube-system/openebs-zfs-node-bwxzr" podUID=2a0727c6-327f-47a2-8bb9-c8741fd9f6ed
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"controller-84d6d4db45-2jc2g_metallb-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"242211d179798ed017f0c5ef4fa41f9219b7819ab04ead8cd70571edfb6ec6f2\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"controller-84d6d4db45-2jc2g_metallb-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"4082d02e09502b0f146c69afae78946220a91b86e6d920cd54693c8e9d3713ba\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.656698    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="metallb-system/controller-84d6d4db45-2jc2g" podUID=24978195-ae80-42c9-80e3-6e880be2bac2
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"aad36eaac89b80ce5df6ed0f54bc7163108ea42b19df455dfbd4ba1de2f3a144\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"dbfc0b44cb38fe556e4fc4fdc53702c0ea5c56e755484332294d2c556ddf79d2\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.662761    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=d03d171f-7cf6-4f50-878b-5cb4e9023b3e
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"intel-gpu-plugin-zvqhc_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"f6851aee517b1b8ac03036b3f5cbfbbbaa550b4b3a48d05c3d1a402c15fccbea\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"intel-gpu-plugin-zvqhc_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"489d3f546c31415786266c5343f3de1bdd4bed99f204bfeec1c59b570ba13ec6\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.668724    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/intel-gpu-plugin-zvqhc" podUID=ee2f204d-7d95-4555-b933-b6ba87ba1547
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"netdata-7bc8fbb7b4-8vz7s_ix-netdata\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"083141b01aa9278834ec5349e0cc12f54c6731fabded3ebfe8aa4c7d50a13c50\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"netdata-7bc8fbb7b4-8vz7s_ix-netdata\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"2161fcd66b50229d0f9ef8ab9d58701f9b2642d641b8b2216d8dd36379a458a1\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.682181    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s" podUID=c4438af0-7353-463f-9a23-1d1bd583a23d
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"coredns-75fc8f8fff-dnkbk_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"23508512b7c2cc3342b3135ebfbf5441a2b35a4b65a78ab89db9ae5d72be73ed\""
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Failed to read pod IP from plugin/docker: networkPlugin cni failed on the status hook for pod \"coredns-75fc8f8fff-dnkbk_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"21016ba7a711cca48be36b1bb82c6eff7e206bc019c5d4a035c8855fdb55fdae\""
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.685318    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-75fc8f8fff-dnkbk" podUID=30a4863f-b5c5-44ac-bd90-df544fd65f1b
Apr 11 14:38:04 truenas k3s[6131]: E0411 14:38:04.685357    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq" podUID=c0561815-b5dd-4748-b1d5-fb362bcd5682
Apr 11 14:38:04 truenas k3s[6131]: I0411 14:38:04.823150    6131 request.go:682] Waited for 1.021770025s due to client-side throttling, not priority and fairness, request: POST:https://127.0.0.1:6443/api/v1/namespaces/kube-system/serviceaccounts/coredns/token
Apr 11 14:38:04 truenas k3s[6131]: time="2023-04-11T14:38:04+02:00" level=info msg="Starting /v1, Kind=Node controller"
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.027434    6131 projected.go:290] Couldn't get configMap prometheus-operator/kube-root-ca.crt: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.027587    6131 projected.go:196] Error preparing data for projected volume kube-api-access-sphlt for pod prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.027623    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt podName:c0561815-b5dd-4748-b1d5-fb362bcd5682 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:05.527612941 +0200 CEST m=+8.245527941 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-sphlt" (UniqueName: "kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt") pod "prometheus-operator-5dcffb7cb8-q6xxq" (UID: "c0561815-b5dd-4748-b1d5-fb362bcd5682") : object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.312614    6131 configmap.go:197] Couldn't get configMap kube-system/coredns-custom: object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.312643    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:07.312635579 +0200 CEST m=+10.030550578 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "custom-config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:05 truenas k3s[6131]: time="2023-04-11T14:38:05+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/16ccd5a63c8fb5b26c980773a987f412ad46ba3e77b552a2b2ed9f08a5e63aa5/resolv.conf as [nameserver 8.8.4.4 nameserver 1.1.1.1]"
Apr 11 14:38:05 truenas k3s[6131]: time="2023-04-11T14:38:05+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/4fdf5551e237d6244af5ef96b93ada1f4cde3047a135aa33cf2196003f4fff9f/resolv.conf as [nameserver 8.8.4.4 nameserver 1.1.1.1]"
Apr 11 14:38:05 truenas k3s[6131]: time="2023-04-11T14:38:05+02:00" level=info msg="Starting /v1, Kind=Pod controller"
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.415927    6131 secret.go:192] Couldn't get secret metallb-system/webhook-server-cert: object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.415959    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert podName:24978195-ae80-42c9-80e3-6e880be2bac2 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:07.415950151 +0200 CEST m=+10.133865151 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "cert" (UniqueName: "kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert") pod "controller-84d6d4db45-2jc2g" (UID: "24978195-ae80-42c9-80e3-6e880be2bac2") : object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.415984    6131 configmap.go:197] Couldn't get configMap kube-system/coredns: object "kube-system"/"coredns" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.415999    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:07.415994714 +0200 CEST m=+10.133909714 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.426653    6131 projected.go:290] Couldn't get configMap ix-netdata/kube-root-ca.crt: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.426681    6131 projected.go:196] Error preparing data for projected volume kube-api-access-gd6g7 for pod ix-netdata/netdata-7bc8fbb7b4-8vz7s: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.426709    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7 podName:c4438af0-7353-463f-9a23-1d1bd583a23d nodeName:}" failed. No retries permitted until 2023-04-11 14:38:05.926699721 +0200 CEST m=+8.644614720 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-gd6g7" (UniqueName: "kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7") pod "netdata-7bc8fbb7b4-8vz7s" (UID: "c4438af0-7353-463f-9a23-1d1bd583a23d") : object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.616719    6131 projected.go:290] Couldn't get configMap prometheus-operator/kube-root-ca.crt: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.616752    6131 projected.go:196] Error preparing data for projected volume kube-api-access-sphlt for pod prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: E0411 14:38:05.616785    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt podName:c0561815-b5dd-4748-b1d5-fb362bcd5682 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:06.616774654 +0200 CEST m=+9.334689654 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-sphlt" (UniqueName: "kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt") pod "prometheus-operator-5dcffb7cb8-q6xxq" (UID: "c0561815-b5dd-4748-b1d5-fb362bcd5682") : object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:05 truenas k3s[6131]: time="2023-04-11T14:38:05+02:00" level=info msg="Starting apps/v1, Kind=DaemonSet controller"
Apr 11 14:38:05 truenas k3s[6131]: W0411 14:38:05.908965    6131 controllermanager.go:278] "cloud-node" is disabled
Apr 11 14:38:05 truenas k3s[6131]: W0411 14:38:05.908974    6131 controllermanager.go:278] "cloud-node-lifecycle" is disabled
Apr 11 14:38:05 truenas k3s[6131]: I0411 14:38:05.909356    6131 controllermanager.go:301] Started "service"
Apr 11 14:38:05 truenas k3s[6131]: W0411 14:38:05.909365    6131 controllermanager.go:278] "route" is disabled
Apr 11 14:38:05 truenas k3s[6131]: I0411 14:38:05.909755    6131 controller.go:237] Starting service controller
Apr 11 14:38:05 truenas k3s[6131]: I0411 14:38:05.909768    6131 shared_informer.go:255] Waiting for caches to sync for service
Apr 11 14:38:06 truenas k3s[6131]: I0411 14:38:06.010578    6131 shared_informer.go:262] Caches are synced for service
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.018847    6131 projected.go:290] Couldn't get configMap ix-netdata/kube-root-ca.crt: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.018860    6131 projected.go:196] Error preparing data for projected volume kube-api-access-gd6g7 for pod ix-netdata/netdata-7bc8fbb7b4-8vz7s: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.018889    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7 podName:c4438af0-7353-463f-9a23-1d1bd583a23d nodeName:}" failed. No retries permitted until 2023-04-11 14:38:07.018879527 +0200 CEST m=+9.736794527 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-gd6g7" (UniqueName: "kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7") pod "netdata-7bc8fbb7b4-8vz7s" (UID: "c4438af0-7353-463f-9a23-1d1bd583a23d") : object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.449386    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=d03d171f-7cf6-4f50-878b-5cb4e9023b3e
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.449410    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s" podUID=c4438af0-7353-463f-9a23-1d1bd583a23d
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.449484    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="metallb-system/controller-84d6d4db45-2jc2g" podUID=24978195-ae80-42c9-80e3-6e880be2bac2
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.449543    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/intel-gpu-plugin-zvqhc" podUID=ee2f204d-7d95-4555-b933-b6ba87ba1547
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.449582    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-75fc8f8fff-dnkbk" podUID=30a4863f-b5c5-44ac-bd90-df544fd65f1b
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.449666    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq" podUID=c0561815-b5dd-4748-b1d5-fb362bcd5682
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.622920    6131 projected.go:290] Couldn't get configMap prometheus-operator/kube-root-ca.crt: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.622940    6131 projected.go:196] Error preparing data for projected volume kube-api-access-sphlt for pod prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:06 truenas k3s[6131]: E0411 14:38:06.622971    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt podName:c0561815-b5dd-4748-b1d5-fb362bcd5682 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:08.622959811 +0200 CEST m=+11.340874811 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-sphlt" (UniqueName: "kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt") pod "prometheus-operator-5dcffb7cb8-q6xxq" (UID: "c0561815-b5dd-4748-b1d5-fb362bcd5682") : object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.032131    6131 projected.go:290] Couldn't get configMap ix-netdata/kube-root-ca.crt: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.032151    6131 projected.go:196] Error preparing data for projected volume kube-api-access-gd6g7 for pod ix-netdata/netdata-7bc8fbb7b4-8vz7s: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.032177    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7 podName:c4438af0-7353-463f-9a23-1d1bd583a23d nodeName:}" failed. No retries permitted until 2023-04-11 14:38:09.032168618 +0200 CEST m=+11.750083621 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-gd6g7" (UniqueName: "kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7") pod "netdata-7bc8fbb7b4-8vz7s" (UID: "c4438af0-7353-463f-9a23-1d1bd583a23d") : object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:07 truenas k3s[6131]: I0411 14:38:07.032325    6131 controller.go:616] quota admission added evaluator for: daemonsets.apps
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.333892    6131 configmap.go:197] Couldn't get configMap kube-system/coredns-custom: object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.333932    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:11.333922242 +0200 CEST m=+14.051837243 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "custom-config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.434728    6131 configmap.go:197] Couldn't get configMap kube-system/coredns: object "kube-system"/"coredns" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.434773    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:11.43476344 +0200 CEST m=+14.152678444 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.434812    6131 secret.go:192] Couldn't get secret metallb-system/webhook-server-cert: object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.434830    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert podName:24978195-ae80-42c9-80e3-6e880be2bac2 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:11.43482374 +0200 CEST m=+14.152738740 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "cert" (UniqueName: "kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert") pod "controller-84d6d4db45-2jc2g" (UID: "24978195-ae80-42c9-80e3-6e880be2bac2") : object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:07 truenas k3s[6131]: I0411 14:38:07.722905    6131 prober_manager.go:287] "Failed to trigger a manual run" probe="Readiness"
Apr 11 14:38:07 truenas k3s[6131]: E0411 14:38:07.991716    6131 kubelet.go:2373] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.449843    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=d03d171f-7cf6-4f50-878b-5cb4e9023b3e
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.449891    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq" podUID=c0561815-b5dd-4748-b1d5-fb362bcd5682
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.449934    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s" podUID=c4438af0-7353-463f-9a23-1d1bd583a23d
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.449985    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-75fc8f8fff-dnkbk" podUID=30a4863f-b5c5-44ac-bd90-df544fd65f1b
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.450022    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/intel-gpu-plugin-zvqhc" podUID=ee2f204d-7d95-4555-b933-b6ba87ba1547
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.450051    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="metallb-system/controller-84d6d4db45-2jc2g" podUID=24978195-ae80-42c9-80e3-6e880be2bac2
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.642623    6131 projected.go:290] Couldn't get configMap prometheus-operator/kube-root-ca.crt: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.642655    6131 projected.go:196] Error preparing data for projected volume kube-api-access-sphlt for pod prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:08 truenas k3s[6131]: E0411 14:38:08.642685    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt podName:c0561815-b5dd-4748-b1d5-fb362bcd5682 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:12.642676159 +0200 CEST m=+15.360591159 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-sphlt" (UniqueName: "kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt") pod "prometheus-operator-5dcffb7cb8-q6xxq" (UID: "c0561815-b5dd-4748-b1d5-fb362bcd5682") : object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:09 truenas k3s[6131]: I0411 14:38:09.026603    6131 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: zfs.csi.openebs.io endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock versions: 1.0.0
Apr 11 14:38:09 truenas k3s[6131]: I0411 14:38:09.026721    6131 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: zfs.csi.openebs.io at endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
Apr 11 14:38:09 truenas k3s[6131]: E0411 14:38:09.044504    6131 projected.go:290] Couldn't get configMap ix-netdata/kube-root-ca.crt: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:09 truenas k3s[6131]: E0411 14:38:09.044519    6131 projected.go:196] Error preparing data for projected volume kube-api-access-gd6g7 for pod ix-netdata/netdata-7bc8fbb7b4-8vz7s: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:09 truenas k3s[6131]: E0411 14:38:09.044577    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7 podName:c4438af0-7353-463f-9a23-1d1bd583a23d nodeName:}" failed. No retries permitted until 2023-04-11 14:38:13.044568689 +0200 CEST m=+15.762483689 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-gd6g7" (UniqueName: "kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7") pod "netdata-7bc8fbb7b4-8vz7s" (UID: "c4438af0-7353-463f-9a23-1d1bd583a23d") : object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:09 truenas k3s[6131]: I0411 14:38:09.245882    6131 controller.go:616] quota admission added evaluator for: zfsnodes.zfs.openebs.io
Apr 11 14:38:09 truenas k3s[6131]: time="2023-04-11T14:38:09+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:10 truenas k3s[6131]: E0411 14:38:10.449835    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-75fc8f8fff-dnkbk" podUID=30a4863f-b5c5-44ac-bd90-df544fd65f1b
Apr 11 14:38:10 truenas k3s[6131]: E0411 14:38:10.449916    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s" podUID=c4438af0-7353-463f-9a23-1d1bd583a23d
Apr 11 14:38:10 truenas k3s[6131]: E0411 14:38:10.449961    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/intel-gpu-plugin-zvqhc" podUID=ee2f204d-7d95-4555-b933-b6ba87ba1547
Apr 11 14:38:10 truenas k3s[6131]: E0411 14:38:10.450010    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq" podUID=c0561815-b5dd-4748-b1d5-fb362bcd5682
Apr 11 14:38:10 truenas k3s[6131]: E0411 14:38:10.450093    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="metallb-system/controller-84d6d4db45-2jc2g" podUID=24978195-ae80-42c9-80e3-6e880be2bac2
Apr 11 14:38:10 truenas k3s[6131]: E0411 14:38:10.450170    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=d03d171f-7cf6-4f50-878b-5cb4e9023b3e
Apr 11 14:38:11 truenas k3s[6131]: E0411 14:38:11.365232    6131 configmap.go:197] Couldn't get configMap kube-system/coredns-custom: object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:11 truenas k3s[6131]: E0411 14:38:11.365261    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:19.365253982 +0200 CEST m=+22.083168981 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "custom-config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-custom-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns-custom" not registered
Apr 11 14:38:11 truenas k3s[6131]: E0411 14:38:11.466009    6131 configmap.go:197] Couldn't get configMap kube-system/coredns: object "kube-system"/"coredns" not registered
Apr 11 14:38:11 truenas k3s[6131]: E0411 14:38:11.466057    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume podName:30a4863f-b5c5-44ac-bd90-df544fd65f1b nodeName:}" failed. No retries permitted until 2023-04-11 14:38:19.46604866 +0200 CEST m=+22.183963659 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "config-volume" (UniqueName: "kubernetes.io/configmap/30a4863f-b5c5-44ac-bd90-df544fd65f1b-config-volume") pod "coredns-75fc8f8fff-dnkbk" (UID: "30a4863f-b5c5-44ac-bd90-df544fd65f1b") : object "kube-system"/"coredns" not registered
Apr 11 14:38:11 truenas k3s[6131]: E0411 14:38:11.466009    6131 secret.go:192] Couldn't get secret metallb-system/webhook-server-cert: object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:11 truenas k3s[6131]: E0411 14:38:11.466078    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert podName:24978195-ae80-42c9-80e3-6e880be2bac2 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:19.466073028 +0200 CEST m=+22.183988028 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "cert" (UniqueName: "kubernetes.io/secret/24978195-ae80-42c9-80e3-6e880be2bac2-cert") pod "controller-84d6d4db45-2jc2g" (UID: "24978195-ae80-42c9-80e3-6e880be2bac2") : object "metallb-system"/"webhook-server-cert" not registered
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.450181    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/intel-gpu-plugin-zvqhc" podUID=ee2f204d-7d95-4555-b933-b6ba87ba1547
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.450253    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=d03d171f-7cf6-4f50-878b-5cb4e9023b3e
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.450302    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="metallb-system/controller-84d6d4db45-2jc2g" podUID=24978195-ae80-42c9-80e3-6e880be2bac2
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.450340    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="ix-netdata/netdata-7bc8fbb7b4-8vz7s" podUID=c4438af0-7353-463f-9a23-1d1bd583a23d
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.450369    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq" podUID=c0561815-b5dd-4748-b1d5-fb362bcd5682
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.450398    6131 pod_workers.go:965] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-75fc8f8fff-dnkbk" podUID=30a4863f-b5c5-44ac-bd90-df544fd65f1b
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.674209    6131 projected.go:290] Couldn't get configMap prometheus-operator/kube-root-ca.crt: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.674241    6131 projected.go:196] Error preparing data for projected volume kube-api-access-sphlt for pod prometheus-operator/prometheus-operator-5dcffb7cb8-q6xxq: object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:12 truenas k3s[6131]: E0411 14:38:12.674287    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt podName:c0561815-b5dd-4748-b1d5-fb362bcd5682 nodeName:}" failed. No retries permitted until 2023-04-11 14:38:20.674277684 +0200 CEST m=+23.392192684 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-sphlt" (UniqueName: "kubernetes.io/projected/c0561815-b5dd-4748-b1d5-fb362bcd5682-kube-api-access-sphlt") pod "prometheus-operator-5dcffb7cb8-q6xxq" (UID: "c0561815-b5dd-4748-b1d5-fb362bcd5682") : object "prometheus-operator"/"kube-root-ca.crt" not registered
Apr 11 14:38:13 truenas k3s[6131]: E0411 14:38:13.076290    6131 projected.go:290] Couldn't get configMap ix-netdata/kube-root-ca.crt: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:13 truenas k3s[6131]: E0411 14:38:13.076305    6131 projected.go:196] Error preparing data for projected volume kube-api-access-gd6g7 for pod ix-netdata/netdata-7bc8fbb7b4-8vz7s: object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:13 truenas k3s[6131]: E0411 14:38:13.076355    6131 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7 podName:c4438af0-7353-463f-9a23-1d1bd583a23d nodeName:}" failed. No retries permitted until 2023-04-11 14:38:21.076323787 +0200 CEST m=+23.794238787 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-gd6g7" (UniqueName: "kubernetes.io/projected/c4438af0-7353-463f-9a23-1d1bd583a23d-kube-api-access-gd6g7") pod "netdata-7bc8fbb7b4-8vz7s" (UID: "c4438af0-7353-463f-9a23-1d1bd583a23d") : object "ix-netdata"/"kube-root-ca.crt" not registered
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.017795    6131 range_allocator.go:76] Sending events to api server.
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.017873    6131 range_allocator.go:110] No Secondary Service CIDR provided. Skipping filtering out secondary service addresses.
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.017996    6131 controllermanager.go:603] Started "nodeipam"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.018058    6131 node_ipam_controller.go:154] Starting ipam controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.018063    6131 shared_informer.go:255] Waiting for caches to sync for node
Apr 11 14:38:14 truenas k3s[6131]: E0411 14:38:14.019366    6131 core.go:218] failed to start cloud node lifecycle controller: no cloud provider provided
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.019377    6131 controllermanager.go:581] Skipping "cloud-node-lifecycle"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.020686    6131 controllermanager.go:603] Started "replicaset"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.020754    6131 replica_set.go:205] Starting replicaset controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.020759    6131 shared_informer.go:255] Waiting for caches to sync for ReplicaSet
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.021938    6131 controllermanager.go:603] Started "ttl"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.022117    6131 ttl_controller.go:120] Starting TTL controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.022124    6131 shared_informer.go:255] Waiting for caches to sync for TTL
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.023245    6131 controllermanager.go:603] Started "deployment"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.023365    6131 deployment_controller.go:160] "Starting controller" controller="deployment"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.023372    6131 shared_informer.go:255] Waiting for caches to sync for deployment
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.024371    6131 controllermanager.go:603] Started "csrcleaner"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.024609    6131 cleaner.go:82] Starting CSR cleaner controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.025579    6131 controllermanager.go:603] Started "root-ca-cert-publisher"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.025617    6131 publisher.go:107] Starting root CA certificate configmap publisher
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.025623    6131 shared_informer.go:255] Waiting for caches to sync for crt configmap
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.026932    6131 controllermanager.go:603] Started "endpointslice"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.027153    6131 endpointslice_controller.go:261] Starting endpoint slice controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.027160    6131 shared_informer.go:255] Waiting for caches to sync for endpoint_slice
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036429    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for horizontalpodautoscalers.autoscaling
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036726    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsrestores.zfs.openebs.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036741    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for l2advertisements.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036754    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for roles.rbac.authorization.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036764    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for csistoragecapacities.storage.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036774    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for probes.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036784    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for alertmanagerconfigs.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036798    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for deployments.apps
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036814    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for podmonitors.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036823    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsnodes.zfs.openebs.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036833    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for communities.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036853    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for addresspools.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036865    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ipaddresspools.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036874    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for middlewares.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036883    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for middlewaretcps.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036891    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpoints
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036901    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for network-attachment-definitions.k8s.cni.cncf.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036909    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressrouteudps.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036918    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpointslices.discovery.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036929    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for backups.postgresql.cnpg.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036938    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsvolumes.zfs.openebs.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036952    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for servicemonitors.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.036961    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for prometheusrules.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037059    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingresses.networking.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037072    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for networkpolicies.networking.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037082    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for alertmanagers.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037091    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsbackups.zfs.openebs.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037100    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for bgpadvertisements.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037114    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for thanosrulers.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037146    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for poolers.postgresql.cnpg.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037157    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serverstransports.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037175    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for statefulsets.apps
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037184    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for clusters.postgresql.cnpg.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037193    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for scheduledbackups.postgresql.cnpg.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037202    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for bgppeers.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037213    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for limitranges
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037224    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for bfdprofiles.metallb.io
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.037306    6131 shared_informer.go:533] resyncPeriod 13h15m49.809200998s is smaller than resyncCheckPeriod 19h18m28.326443493s and the informer has already started. Changing it to 19h18m28.326443493s
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037336    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serviceaccounts
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037353    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmchartconfigs.helm.cattle.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037362    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for prometheuses.monitoring.coreos.com
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037371    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressroutes.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037381    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for poddisruptionbudgets.policy
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037391    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for rolebindings.rbac.authorization.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037399    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for leases.coordination.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037408    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmcharts.helm.cattle.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037418    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for addons.k3s.cattle.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037427    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for tlsstores.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037438    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for tlsoptions.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037453    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for daemonsets.apps
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037466    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for controllerrevisions.apps
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037476    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for replicasets.apps
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037485    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for jobs.batch
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037495    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for volumesnapshots.snapshot.storage.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037504    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for traefikservices.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037513    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for podtemplates
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037522    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for cronjobs.batch
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037531    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfssnapshots.zfs.openebs.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037547    6131 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressroutetcps.traefik.containo.us
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037565    6131 controllermanager.go:603] Started "resourcequota"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037766    6131 resource_quota_controller.go:277] Starting resource quota controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.037772    6131 shared_informer.go:255] Waiting for caches to sync for resource quota
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.038086    6131 resource_quota_monitor.go:295] QuotaMonitor running
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.039121    6131 controllermanager.go:603] Started "serviceaccount"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.040263    6131 serviceaccounts_controller.go:117] Starting service account controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.040274    6131 shared_informer.go:255] Waiting for caches to sync for service account
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.040818    6131 controllermanager.go:603] Started "daemonset"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.041137    6131 daemon_controller.go:291] Starting daemon sets controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.041146    6131 shared_informer.go:255] Waiting for caches to sync for daemon sets
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.042960    6131 controllermanager.go:603] Started "endpoint"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.043162    6131 endpoints_controller.go:182] Starting endpoint controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.043170    6131 shared_informer.go:255] Waiting for caches to sync for endpoint
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.046535    6131 controllermanager.go:603] Started "garbagecollector"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.048799    6131 controllermanager.go:603] Started "disruption"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.049253    6131 disruption.go:421] Sending events to api server.
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.049273    6131 disruption.go:432] Starting disruption controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.049285    6131 shared_informer.go:255] Waiting for caches to sync for disruption
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.050520    6131 controllermanager.go:603] Started "statefulset"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.050704    6131 stateful_set.go:152] Starting stateful set controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.050711    6131 shared_informer.go:255] Waiting for caches to sync for stateful set
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.051864    6131 controllermanager.go:603] Started "pvc-protection"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.051946    6131 pvc_protection_controller.go:103] "Starting PVC protection controller"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.051952    6131 shared_informer.go:255] Waiting for caches to sync for PVC protection
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.053339    6131 controllermanager.go:603] Started "attachdetach"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.053445    6131 attach_detach_controller.go:328] Starting attach detach controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.053451    6131 shared_informer.go:255] Waiting for caches to sync for attach detach
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.054498    6131 controllermanager.go:603] Started "clusterrole-aggregation"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.054576    6131 clusterroleaggregation_controller.go:194] Starting ClusterRoleAggregator
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.054582    6131 shared_informer.go:255] Waiting for caches to sync for ClusterRoleAggregator
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.066190    6131 controllermanager.go:603] Started "namespace"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.066285    6131 namespace_controller.go:200] Starting namespace controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.066292    6131 shared_informer.go:255] Waiting for caches to sync for namespace
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.067601    6131 controllermanager.go:603] Started "job"
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.067609    6131 controllermanager.go:568] "bootstrapsigner" is disabled
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.067735    6131 job_controller.go:196] Starting job controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.067741    6131 shared_informer.go:255] Waiting for caches to sync for job
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.074255    6131 garbagecollector.go:154] Starting garbage collector controller
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.074270    6131 shared_informer.go:255] Waiting for caches to sync for garbage collector
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.074300    6131 graph_builder.go:291] GraphBuilder running
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.082001    6131 shared_informer.go:255] Waiting for caches to sync for garbage collector
Apr 11 14:38:14 truenas k3s[6131]: E0411 14:38:14.113050    6131 core.go:92] Failed to start service controller: WARNING: no cloud provider provided, services of type LoadBalancer will fail
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.113132    6131 controllermanager.go:581] Skipping "service"
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.113141    6131 core.go:232] configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.113147    6131 controllermanager.go:581] Skipping "route"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.117245    6131 shared_informer.go:255] Waiting for caches to sync for resource quota
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.135301    6131 actual_state_of_world.go:541] Failed to update statusUpdateNeeded field in actual state of world: Failed to set statusUpdateNeeded to needed true, because nodeName="ix-truenas" does not exist
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.142489    6131 shared_informer.go:262] Caches are synced for GC
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.142553    6131 shared_informer.go:262] Caches are synced for endpoint_slice_mirroring
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.143190    6131 shared_informer.go:262] Caches are synced for endpoint
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.149605    6131 shared_informer.go:262] Caches are synced for PV protection
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.150679    6131 shared_informer.go:262] Caches are synced for ReplicationController
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.150819    6131 shared_informer.go:262] Caches are synced for stateful set
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.152670    6131 shared_informer.go:262] Caches are synced for certificate-csrapproving
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.152696    6131 shared_informer.go:262] Caches are synced for PVC protection
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.155641    6131 shared_informer.go:262] Caches are synced for ClusterRoleAggregator
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.167772    6131 shared_informer.go:262] Caches are synced for job
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.176265    6131 shared_informer.go:262] Caches are synced for persistent volume
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.199453    6131 shared_informer.go:262] Caches are synced for expand
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.200635    6131 shared_informer.go:262] Caches are synced for TTL after finished
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.201860    6131 shared_informer.go:262] Caches are synced for ephemeral
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.203421    6131 shared_informer.go:262] Caches are synced for cronjob
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.210407    6131 shared_informer.go:262] Caches are synced for certificate-csrsigning-legacy-unknown
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.210437    6131 shared_informer.go:262] Caches are synced for certificate-csrsigning-kubelet-serving
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.210454    6131 shared_informer.go:262] Caches are synced for certificate-csrsigning-kubelet-client
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.210465    6131 shared_informer.go:262] Caches are synced for certificate-csrsigning-kube-apiserver-client
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.218474    6131 shared_informer.go:262] Caches are synced for node
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.218592    6131 range_allocator.go:166] Starting range CIDR allocator
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.218649    6131 shared_informer.go:255] Waiting for caches to sync for cidrallocator
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.218691    6131 shared_informer.go:262] Caches are synced for cidrallocator
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.220917    6131 shared_informer.go:262] Caches are synced for ReplicaSet
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.222436    6131 shared_informer.go:262] Caches are synced for TTL
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.225719    6131 shared_informer.go:262] Caches are synced for crt configmap
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.228138    6131 shared_informer.go:262] Caches are synced for endpoint_slice
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.272781    6131 controller.go:616] quota admission added evaluator for: endpoints
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.347352    6131 shared_informer.go:262] Caches are synced for HPA
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.353935    6131 shared_informer.go:262] Caches are synced for attach detach
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.530795    6131 controller.go:616] quota admission added evaluator for: endpointslices.discovery.k8s.io
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.542229    6131 shared_informer.go:262] Caches are synced for daemon sets
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.547579    6131 shared_informer.go:262] Caches are synced for taint
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.548075    6131 node_lifecycle_controller.go:1443] Initializing eviction metric for zone:
Apr 11 14:38:14 truenas k3s[6131]: W0411 14:38:14.548490    6131 node_lifecycle_controller.go:1058] Missing timestamp for Node ix-truenas. Assuming now as a timestamp.
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.548684    6131 node_lifecycle_controller.go:1259] Controller detected that zone  is now in state Normal.
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.548988    6131 taint_manager.go:204] "Starting NoExecuteTaintManager"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.549052    6131 taint_manager.go:209] "Sending events to api server"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.549420    6131 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="v1" type="Normal" reason="RegisteredNode" message="Node ix-truenas event: Registered Node ix-truenas in Controller"
Apr 11 14:38:14 truenas k3s[6131]: time="2023-04-11T14:38:14+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.706165    6131 scope.go:115] "RemoveContainer" containerID="390bc00f5f34b472fc134c764fe3493aea08eddbbfcdec466e286cad67dfbedc"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.723453    6131 shared_informer.go:262] Caches are synced for deployment
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.740404    6131 shared_informer.go:262] Caches are synced for service account
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.749802    6131 shared_informer.go:262] Caches are synced for disruption
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.766320    6131 shared_informer.go:262] Caches are synced for namespace
Apr 11 14:38:14 truenas k3s[6131]: E0411 14:38:14.767994    6131 remote_runtime.go:599] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 390bc00f5f34b472fc134c764fe3493aea08eddbbfcdec466e286cad67dfbedc" containerID="390bc00f5f34b472fc134c764fe3493aea08eddbbfcdec466e286cad67dfbedc"
Apr 11 14:38:14 truenas k3s[6131]: E0411 14:38:14.768014    6131 kuberuntime_manager.go:1024] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 390bc00f5f34b472fc134c764fe3493aea08eddbbfcdec466e286cad67dfbedc" pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:14 truenas k3s[6131]: E0411 14:38:14.768091    6131 generic.go:411] "PLEG: Write status" err="rpc error: code = Unknown desc = Error: No such container: 390bc00f5f34b472fc134c764fe3493aea08eddbbfcdec466e286cad67dfbedc" pod="kube-system/intel-gpu-plugin-zvqhc"
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.783068    6131 shared_informer.go:262] Caches are synced for garbage collector
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.783127    6131 shared_informer.go:262] Caches are synced for garbage collector
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.783132    6131 garbagecollector.go:163] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.817336    6131 shared_informer.go:262] Caches are synced for resource quota
Apr 11 14:38:14 truenas k3s[6131]: I0411 14:38:14.838472    6131 shared_informer.go:262] Caches are synced for resource quota
Apr 11 14:38:15 truenas k3s[6131]: time="2023-04-11T14:38:15+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/45750e97d5797f1551114313a00bee065bc9a82a823f8a8c5fba8d08e1f657dd/resolv.conf as [nameserver 172.17.0.10 search kube-system.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Apr 11 14:38:15 truenas k3s[6131]: time="2023-04-11T14:38:15+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/9741c63626636e14d714c69e80d45257ddd8f5feb357981ded09af3a44aae6c4/resolv.conf as [nameserver 172.17.0.10 search kube-system.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Apr 11 14:38:15 truenas k3s[6131]: I0411 14:38:15.490050    9517 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"openebs-zfs-controller-0", UID:"d03d171f-7cf6-4f50-878b-5cb4e9023b3e", APIVersion:"v1", ResourceVersion:"1410304", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.58/16] from ix-net
Apr 11 14:38:15 truenas k3s[6131]: I0411 14:38:15.507350    9497 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"intel-gpu-plugin-zvqhc", UID:"ee2f204d-7d95-4555-b933-b6ba87ba1547", APIVersion:"v1", ResourceVersion:"1410307", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.57/16] from ix-net
Apr 11 14:38:17 truenas k3s[6131]: I0411 14:38:17.818792    6131 scope.go:115] "RemoveContainer" containerID="7606d7e4c9fbfe415d160e6e6b9c25ef0e6df71e51076fefedff20364c1bb68f"
Apr 11 14:38:19 truenas k3s[6131]: time="2023-04-11T14:38:19+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:20 truenas k3s[6131]: time="2023-04-11T14:38:20+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/5989dc10707167aae2fd7d7545fe85b0b5116642c86248e1d31e0306700aa1dc/resolv.conf as [nameserver 172.17.0.10 search metallb-system.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Apr 11 14:38:20 truenas k3s[6131]: time="2023-04-11T14:38:20+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/d2673f5397c641cee91b8afd3f680be48e722b25c89d0f7978870beba67240ca/resolv.conf as [nameserver 8.8.4.4 nameserver 1.1.1.1]"
Apr 11 14:38:20 truenas k3s[6131]: I0411 14:38:20.379972    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="d2673f5397c641cee91b8afd3f680be48e722b25c89d0f7978870beba67240ca"
Apr 11 14:38:20 truenas k3s[6131]: I0411 14:38:20.383476    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="5989dc10707167aae2fd7d7545fe85b0b5116642c86248e1d31e0306700aa1dc"
Apr 11 14:38:20 truenas k3s[6131]: I0411 14:38:20.452696   10649 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"metallb-system", Name:"controller-84d6d4db45-2jc2g", UID:"24978195-ae80-42c9-80e3-6e880be2bac2", APIVersion:"v1", ResourceVersion:"1410301", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.59/16] from ix-net
Apr 11 14:38:20 truenas k3s[6131]: I0411 14:38:20.489260   10674 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"coredns-75fc8f8fff-dnkbk", UID:"30a4863f-b5c5-44ac-bd90-df544fd65f1b", APIVersion:"v1", ResourceVersion:"1410321", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.60/16] from ix-net
Apr 11 14:38:20 truenas k3s[6131]: I0411 14:38:20.522995    6131 server.go:142] "Got registration request from device plugin with resource" resourceName="gpu.intel.com/i915"
Apr 11 14:38:21 truenas k3s[6131]: time="2023-04-11T14:38:21+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/83da8920ea9085a766bc1ce2cb9e026dd2ad245d34a699004e2343630e319a52/resolv.conf as [nameserver 172.17.0.10 search prometheus-operator.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Apr 11 14:38:21 truenas k3s[6131]: I0411 14:38:21.650024    6131 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="83da8920ea9085a766bc1ce2cb9e026dd2ad245d34a699004e2343630e319a52"
Apr 11 14:38:21 truenas k3s[6131]: I0411 14:38:21.761744   11078 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"prometheus-operator", Name:"prometheus-operator-5dcffb7cb8-q6xxq", UID:"c0561815-b5dd-4748-b1d5-fb362bcd5682", APIVersion:"v1", ResourceVersion:"1410286", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.61/16] from ix-net
Apr 11 14:38:22 truenas k3s[6131]: time="2023-04-11T14:38:22+02:00" level=info msg="Will attempt to re-write config file /mnt/RaidZ1/ix-applications/docker/containers/10222db1251fa8d9e0bdf173b2f51a7126f3cc6551a0fca5bfcb6260c4116518/resolv.conf as [nameserver 172.17.0.10 search ix-netdata.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Apr 11 14:38:22 truenas k3s[6131]: I0411 14:38:22.497607   11354 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-netdata", Name:"netdata-7bc8fbb7b4-8vz7s", UID:"c4438af0-7353-463f-9a23-1d1bd583a23d", APIVersion:"v1", ResourceVersion:"1410318", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.62/16] from ix-net
Apr 11 14:38:24 truenas k3s[6131]: time="2023-04-11T14:38:24+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:29 truenas k3s[6131]: time="2023-04-11T14:38:29+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:30 truenas k3s[6131]: {"level":"warn","ts":"2023-04-11T14:38:30.479+0200","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000761180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Apr 11 14:38:34 truenas k3s[6131]: time="2023-04-11T14:38:34+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:39 truenas k3s[6131]: time="2023-04-11T14:38:39+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:44 truenas k3s[6131]: time="2023-04-11T14:38:44+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:49 truenas k3s[6131]: time="2023-04-11T14:38:49+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:54 truenas k3s[6131]: time="2023-04-11T14:38:54+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:38:59 truenas k3s[6131]: time="2023-04-11T14:38:59+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:02 truenas k3s[6131]: {"level":"warn","ts":"2023-04-11T14:39:02.913+0200","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000761180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Apr 11 14:39:04 truenas k3s[6131]: time="2023-04-11T14:39:04+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:09 truenas k3s[6131]: time="2023-04-11T14:39:09+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:14 truenas k3s[6131]: time="2023-04-11T14:39:14+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:19 truenas k3s[6131]: time="2023-04-11T14:39:19+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:24 truenas k3s[6131]: time="2023-04-11T14:39:24+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:29 truenas k3s[6131]: time="2023-04-11T14:39:29+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:34 truenas k3s[6131]: time="2023-04-11T14:39:34+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:39 truenas k3s[6131]: time="2023-04-11T14:39:39+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:42 truenas k3s[6131]: {"level":"warn","ts":"2023-04-11T14:39:42.615+0200","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000761180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Apr 11 14:39:44 truenas k3s[6131]: time="2023-04-11T14:39:44+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:49 truenas k3s[6131]: time="2023-04-11T14:39:49+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:54 truenas k3s[6131]: time="2023-04-11T14:39:54+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:39:59 truenas k3s[6131]: time="2023-04-11T14:39:59+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:04 truenas k3s[6131]: time="2023-04-11T14:40:04+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:09 truenas k3s[6131]: time="2023-04-11T14:40:09+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:14 truenas k3s[6131]: time="2023-04-11T14:40:14+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:19 truenas k3s[6131]: time="2023-04-11T14:40:19+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:24 truenas k3s[6131]: time="2023-04-11T14:40:24+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:26 truenas k3s[6131]: {"level":"warn","ts":"2023-04-11T14:40:26.261+0200","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000761180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Apr 11 14:40:29 truenas k3s[6131]: time="2023-04-11T14:40:29+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:34 truenas k3s[6131]: time="2023-04-11T14:40:34+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Apr 11 14:40:39 truenas k3s[6131]: time="2023-04-11T14:40:39+02:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
debug finished in 0 seconds for K3s Logs (last 1000 lines)


