Skip to content

Latest commit

 

History

History
741 lines (662 loc) · 32.9 KB

tidb2es.org

File metadata and controls

741 lines (662 loc) · 32.9 KB
Raw

EKS SETUP

Helm setup

Install v3.8.2 because the latest version has bugs.

OhMyTiUP$ curl -L https://git.io/get_helm.sh | bash -s -- --version v3.8.2

Cluster install

ohMyTiUP$ more eks.cluster.yaml
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
metadata:
  name: elaticcluster
  region: us-east-1

nodeGroups:
  - name: admin
    desiredCapacity: 1
    privateNetworking: true
    labels:
      dedicated: admin

  - name: elastic
    desiredCapacity: 3
    privateNetworking: true
    instanceType: c5.2xlarge
    labels:
      dedicated: elastic
    taints:
      dedicated: elastic:NoSchedule
OhMyTiUP$ eksctl create cluster -f eks.config.yaml
admin@OhMyTiUP:~$ kubectl get nodes
NAME                             STATUS   ROLES    AGE   VERSION
ip-192-168-67-129.ec2.internal   Ready    <none>   41h   v1.23.13-eks-fb459a0
ip-192-168-72-129.ec2.internal   Ready    <none>   41h   v1.23.13-eks-fb459a0
ip-192-168-79-24.ec2.internal    Ready    <none>   41h   v1.23.13-eks-fb459a0
ip-192-168-96-59.ec2.internal    Ready    <none>   41h   v1.23.13-eks-fb459a0
OhMyTiUP$  eksctl get nodegroup --cluster elaticcluster
CLUSTER         NODEGROUP       STATUS          CREATED                 MIN SIZE        MAX SIZE        DESIRED CAPACITY        INSTANCE TYPE   IMAGE ID                ASG NAME    TYPE
elaticcluster   admin           CREATE_COMPLETE 2022-12-12T02:00:00Z    1               1               1                       m5.large        ami-0c9424a408e18bcc9   eksctl-elaticcluster-nodegroup-admin-NodeGroup-1RFOL3A8LDJKU unmanaged
elaticcluster   elastic         CREATE_COMPLETE 2022-12-12T02:00:00Z    3               3               3                       c5.2xlarge      ami-0c9424a408e18bcc9   eksctl-elaticcluster-nodegroup-elastic-NodeGroup-1BS6RFYX18IEX       unmanaged

dynamic volume provisioning

OhMyTiUP$ eksctl create iamserviceaccount \
   --name ebs-csi-controller-sa \
   --namespace kube-system \
   --cluster elaticcluster \
   --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy   \
   --approve   --role-only   --role-name AmazonEKS_EBS_CSI_DriverRole
OhMyTiUP$ eksctl get iamserviceaccount --cluster elaticcluster
NAMESPACE       NAME                    ROLE ARN
kube-system     ebs-csi-controller-sa   arn:aws:iam::729581434105:role/AmazonEKS_EBS_CSI_DriverRole
OhMyTiUP$ eksctl create addon --name aws-ebs-csi-driver \
    --cluster elaticcluster --service-account-role-arn arn:aws:iam::729581434105:role/AmazonEKS_EBS_CSI_DriverRole --force
OhMyTiUP$ eksctl get addon --cluster elaticcluster
NAME                    VERSION                 STATUS  ISSUES  IAMROLE                                                         UPDATE AVAILABLE
aws-ebs-csi-driver      v1.13.0-eksbuild.3      ACTIVE  0       arn:aws:iam::729581434105:role/AmazonEKS_EBS_CSI_DriverRole

aws eks describe-addon-versions --addon-name aws-ebs-csi-driver

nginx ingress installation

OhMyTiUP$ helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
OhMyTiUP$ helm repo update
OhMyTiUP$ helm install nginx-ingress-controller ingress-nginx/ingress-nginx

storageClass setup

OhMyTiUP$ more storageClass.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: es-gp2
provisioner: ebs.csi.aws.com # Amazon EBS CSI driver
parameters:
  type: gp2
  encrypted: 'true' # EBS volumes will always be encrypted by default
volumeBindingMode: WaitForFirstConsumer # EBS volumes are AZ specific
reclaimPolicy: Delete
mountOptions:
- debug
OhMyTiUP$ kubectl create -f storageClass.yaml
OhMyTiUP$ kubectl get storageClass 
NAME            PROVISIONER             RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
es-gp2          ebs.csi.aws.com         Delete          WaitForFirstConsumer   false                  56s
gp2 (default)   kubernetes.io/aws-ebs   Delete          WaitForFirstConsumer   false                  9h

Elasticsearch install

Elasticsearch config file preparation

Please refer to Elticsearch doc for elasticsearch installation.

OhMyTiUP$ helm repo add bitnami https://charts.bitnami.com/bitnami
OhMyTiUP$ helm repo add elastic https://helm.elastic.co
OhMyTiUP$ more Values.yaml
antiAffinity: "soft"

# Shrink default JVM heap.
esJavaOpts: "-Xmx128m -Xms128m"

# Allocate smaller chunks of memory per pod.
resources:
  requests:
    cpu: "100m"
    memory: "512M"
  limits:
    cpu: "1000m"
    memory: "512M"

secret:
  enabled: true
  password: "1234Abcd"

volumeClaimTemplate:
  accessModes: [ "ReadWriteOnce" ]
  storageClassName: "es-gp2"
  resources:
    requests:
      storage: 100M

OhMyTiUP$ helm install elasticsearch elastic/elasticsearch -f Values.yaml

ingress setup

OhMyTiUP$ more ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: hello-world
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
  ingressClassName: nginx
  defaultBackend:
    service:
      name: elasticsearch-master
      port:
        number: 9200
  rules:
  - http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: elasticsearch-master
            port:
              number: 9200
OhMyTiUP$ kubectl create -f ingress.yaml
OhMyTiUP$ kubectl get ingress 
NAME          CLASS   HOSTS   ADDRESS                                                                   PORTS   AGE
hello-world   nginx   *       a0c6807e4db744186be5f480ae1a33a2-1781717375.us-east-1.elb.amazonaws.com   80      24s
OhMyTiUP$ kubectl describe ingress hello-world 
Name:             hello-world
Namespace:        default
Address:          a0c6807e4db744186be5f480ae1a33a2-1781717375.us-east-1.elb.amazonaws.com
Default backend:  elasticsearch-master:9200 (<error: endpoints "elasticsearch-master" not found>)
Rules:
  Host        Path  Backends
  ----        ----  --------
  *           
              /   elasticsearch-master:9200 (<error: endpoints "elasticsearch-master" not found>)
Annotations:  nginx.ingress.kubernetes.io/backend-protocol: HTTPS
Events:
  Type    Reason  Age                From                      Message
  ----    ------  ----               ----                      -------
  Normal  Sync    89s (x2 over 97s)  nginx-ingress-controller  Scheduled for sync

https://docs.aws.amazon.com/eks/latest/userguide/csi-iam-role.html

https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md

https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/

https://www.eksworkshop.com/beginner/170_statefulset/storageclass/

https://github.com/elastic/helm-charts/tree/main/elasticsearch/examples

ingress

OhMyTiUP$ more apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: hello-world
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
  ingressClassName: nginx
  defaultBackend:
    service:
      name: elasticsearch-master
      port:
        number: 9200
  rules:
  - http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: elasticsearch-master
            port:
              number: 9200
OhMyTiUP$ kubectl create -f ingress.yaml
admin@ip-172-82-11-66:~/workspace$ kubectl get ingress   
NAME          CLASS   HOSTS   ADDRESS                                                                   PORTS   AGE
hello-world   nginx   *       a4c5a5e5767ac4aff83982c48023a48a-1396949624.us-east-1.elb.amazonaws.com   80      10m
OhMyTiUP$ curl --insecure -u elastic:1234Abcd https://52.20.241.191/

Setup the nginx ingress controller

Setup the nginx ingress controller with internal alb.

ohMyTiUP$ more internal-alb.yaml
controller:
  ingressClassByName: true

  ingressClassResource:
    name: nginx-internal
    enabled: true
    default: false
    controllerValue: "k8s.io/ingress-nginx-internal"

  service:
    # Disable the external LB
    external:
      enabled: false

    # Enable the internal LB. The annotations are important here, without
    # these you will get a "classic" loadbalancer
    internal:
      enabled: true
      annotations:
        service.beta.kubernetes.io/aws-load-balancer-internal: "true"
        service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
        service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true'
        service.beta.kubernetes.io/aws-load-balancer-type: alb
OhMyTiUP$ helm install nginx-ingress-controller ingress-nginx/ingress-nginx -f internal-alb.yaml

Setup the nginx ingress

OhMyTiUP$ more ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: helloworld
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    kubernetes.io/ingress.class: "nginx"
spec:
        #  ingressClassName: nginx
  defaultBackend:
    service:
      name: elasticsearch-master
      port:
        number: 9200
  rules:
  - http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: elasticsearch-master
            port:
              number: 9200
OhMyTiUP$ kubectl create -f ./ingress.yaml

Confirmation

OhMyTiUP$ kubectl get service
NAME                                                          TYPE           CLUSTER-IP       EXTERNAL-IP                                                                        PORT(S)                      AGE
elasticsearch-master                                          ClusterIP      10.100.91.100    <none>                                                                             9200/TCP,9300/TCP            5h28m
elasticsearch-master-headless                                 ClusterIP      None             <none>                                                                             9200/TCP,9300/TCP            5h28m
kubernetes                                                    ClusterIP      10.100.0.1       <none>                                                                             443/TCP                      7d22h
nginx-ingress-controller-ingress-nginx-controller-admission   ClusterIP      10.100.185.203   <none>                                                                             443/TCP                      108m
nginx-ingress-controller-ingress-nginx-controller-internal    LoadBalancer   10.100.176.255   internal-ad7066fdb66964bde95b5149b37d6add-1296352725.us-east-1.elb.amazonaws.com   80:31638/TCP,443:32760/TCP   108m

OhMyTiUP$ curl -k -u elastic:1234Abcd https://internal-ad7066fdb66964bde95b5149b37d6add-1296352725.us-east-1.elb.amazonaws.com/
{
  "name" : "elasticsearch-master-2",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "SmDD_5HRRhSamZeF65fFbQ",
  "version" : {
    "number" : "8.5.1",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "c1310c45fc534583afe2c1c03046491efba2bba2",
    "build_date" : "2022-11-09T21:02:20.169855900Z",
    "build_snapshot" : false,
    "lucene_version" : "9.4.1",
    "minimum_wire_compatibility_version" : "7.17.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "You Know, for Search"
}

Manual setup

OhMyTiUP$ more /tmp/source.toml 
[sink]                                                                                                                                                                               
dispatchers = [
  {matcher = ['*.*'], topic = "{schema}_{table}", partition ="ts"},
]
OhMyTiUP$ tiup cdc cli changefeed create --server http://182.83.3.147:8300 --changefeed-id='tidb2es' --sink-uri=kafka://172.83.6.249:9092/topic-name?protocol=avro --schema-registry=http://172.83.6.222:8081 --config=/tmp/source.toml
Info: {"upstream_id":7186473498337929556,"namespace":"default","id":"tidb2es","sink_uri":"kafka://172.83.6.249:9092/topic-name?protocol=avro","create_time":"2023-01-09T05:26:59.8314
4617Z","start_ts":438630355920486401,"engine":"unified","config":{"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_saf
e_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mo
unter":null,"sink":{"protocol":"avro","schema_registry":"http://172.83.6.222:8081","csv":null,"dispatchers":[{"matcher":["*.*"],"partition":"ts","topic":"{schema}_{table}"}],"column
_selectors":null,"transaction_atomicity":"none","encoder_concurrency":0,"terminator":"","date_separator":"","enable_partition_separator":false},"consistent":{"level":"none","max_log
_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v6.3.0"}
OhMyTiUP$
OhMyTiUP$ tiup cdc cli changefeed list --server http://182.83.3.147:8300
tiup is checking updates for component cdc ...
Starting component `cdc`: /home/admin/.tiup/components/cdc/v6.5.0/cdc cli changefeed list --server http://182.83.3.147:8300
[
  {
    "id": "tidb2es",
    "namespace": "default",
    "summary": {
      "state": "normal",
      "tso": 438630410996678657,
      "checkpoint": "2023-01-09 05:30:29.449",
      "error": null
    }
  }
]

OhMyTiUP$  /opt/kafka/perf/kafka-util.sh list-topic
__consumer_offsets
_schemas
connect-configs
connect-offsets
connect-status
topic-name

OhMyTiUP$ /opt/kafka/perf/kafka-util.sh topic-offset topic-name 
topic-name:0:0
topic-name:1:0
topic-name:2:0
MySQL$ insert into test01 values(2,2);

OhMyTiUP$  /opt/kafka/perf/kafka-util.sh list-topic
__consumer_offsets
_schemas
connect-configs
connect-offsets
connect-status
test_t:q!
est01
topic-name

sink connecto install

without timestamp

OhMyTiUP$ more /tmp/sink.json
{
    "name": "ESSINK",
    "config": {
        "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
        "connection.url": "http://172.89.2.213:80",
        "connection.username": "elastic",
        "connection.password": "1234Abcd",
        "key.ignore": "true",
        "value.converter": "io.confluent.connect.avro.AvroConverter",
        "value.converter.schema.registry.url": "http://172.83.1.189:8081",
        "topics": "test_test01"
    }
}
OhMyTiUP$ curl -X POST -H "content-type:application/json" http://172.83.1.100:8083/connectors -d@'/tmp/sink.json'

OhMyTiUP$ curl http://172.83.1.100:8083/connectors/ESSINK/status | jq 
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   164  100   164    0     0  54666      0 --:--:-- --:--:-- --:--:-- 54666
{
  "name": "ESSINK",
  "connector": {
    "state": "RUNNING",
    "worker_id": "172.83.1.100:8083"
  },
  "tasks": [
    {
      "id": 0,
      "state": "RUNNING",
      "worker_id": "172.83.1.100:8083"
    }
  ],
  "type": "sink"
}
OhMyTiUP$ curl http://172.83.1.100:8083/connectors/ESSINK | jq 
{
  "name": "ESSINK",
  "config": {
    "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
    "value.converter.schema.registry.url": "http://172.83.1.189:8081",
    "connection.password": "1234Abcd",
    "topics": "test_test01",
    "connection.username": "elastic",
    "name": "ESSINK",
    "connection.url": "http://172.89.2.213:80",
    "key.ignore": "true",
    "value.converter": "io.confluent.connect.avro.AvroConverter"
  },
  "tasks": [
    {
      "connector": "ESSINK",
      "task": 0
    }
  ],
  "type": "sink"
}
OhMyTiUP$ curl -u elastic:1234Abcd http://172.89.2.213/test_test01/_count | jq 
{
  "count": 5,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  }
}
OhMyTiUP$ curl -u elastic:1234Abcd http://172.89.2.213/test_test01/_search?q=col01:10 | jq 
{
  "took": 14,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 1,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "test_test01",
        "_id": "test_test01+1+0",
        "_score": 1,
        "_source": {
          "col01": 10,
          "col02": 10
        }
      }
    ]
  }
}

with timestamp

connector$ sudo confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:latest
connector$ sudo systemctl restart confluent-kafka-connect
OhMyTiUP$ curl -X POST -H "content-type:application/json" http://172.83.4.109:8083/connectors -d @'/tmp/sink.json' | jq
{
  "name": "ESSINK",
  "config": {
    "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
    "connection.url": "http://172.89.2.213:80",
    "connection.username": "elastic",
    "connection.password": "1234Abcd",
    "topics": "test_test01",
    "key.converter": "io.confluent.connect.avro.AvroConverter",
    "value.converter": "io.confluent.connect.avro.AvroConverter",
    "key.converter.schema.registry.url": "http://172.83.6.222:8081",
    "value.converter.schema.registry.url": "http://172.83.6.222:8081",
    "key.converter.schemas.enable": "true",
    "value.converter.schemas.enable": "true",
    "transforms": "timestamp",
    "transforms.timestamp.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value",
    "transforms.timestamp.target.type": "Timestamp",
    "transforms.timestamp.field": "timestamp,t_datetime",
    "transforms.timestamp.format": "yyyy-MM-dd HH:mm:ss",
    "name": "ESSINK"
  },
  "tasks": [],
  "type": "sink"
}
MySQL [test]> insert into test01 values(3,3); 
Query OK, 1 row affected (0.005 sec)

admin@ip-172-82-11-40:~$ tiup cdc cli changefeed create --server http://182.83.3.147:8300 --changefeed-id='tidb2es' --sink-uri=kafka://172.83.6.249:9092/topic-name?protocol=canal-json --config=/tmp/source.toml 
tiup is checking updates for component cdc ...
Starting component `cdc`: /home/admin/.tiup/components/cdc/v6.5.0/cdc cli changefeed create --server http://182.83.3.147:8300 --changefeed-id=tidb2es --sink-uri=kafka://172.83.6.249:9092/topic-name?protocol=canal-json --config=/tmp/source.toml
Create changefeed successfully!
ID: tidb2es
Info: {"upstream_id":7186473498337929556,"namespace":"default","id":"tidb2es","sink_uri":"kafka://172.83.6.249:9092/topic-name?protocol=canal-json","create_time":"2023-01-09T12:25:05.766528516Z","start_ts":438636932130144257,"engine":"unified","config":{"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":null,"sink":{"protocol":"canal-json","schema_registry":"","csv":null,"dispatchers":[{"matcher":["*.*"],"partition":"ts","topic":"{schema}_{table}"}],"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":0,"terminator":"","date_separator":"","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v6.3.0"}

  • create the role AmazonEKS_EBS_CSI_DriverRole_ using skd rather than eksctl
  • load balancer: there are two ip which is the most proper one
  • Cleanness
    • role - ekstest
    • volume

curl -Lo aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64

aws eks update-kubeconfig –region region-code –name my-cluster curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl

kubectl apply -f https://docs.projectcalico.org/manifests/calico-typha.yaml

-> ip-172-89-2-252.ec2.internal becomes ready

kubectl get validatingwebhookconfigurations -> kubectl delete validatingwebhookconfigurations nginx-ingress-controller-ingress-nginx-admission

Why do we need to add the label: dedicate:elastic taint: dedicat:elastic:noschedule

DIFF

  admin@ip-172-82-11-48:~/workspace/new-cluster$ eksctl get nodegroup --cluster estest 
CLUSTER NODEGROUP       STATUS  CREATED                 MIN SIZE        MAX SIZE        DESIRED CAPACITY        INSTANCE TYPE   IMAGE ID        ASG NAME                            TYPE
estest  elasticsearch   ACTIVE  2022-12-26T22:37:10Z    3               3               3                       c5.xlarge       AL2_x86_64      eks-elasticsearch-f2c2a84a-9375-7345-9edc-ee6d49d20d34       managed
estest  esNodeGroup     ACTIVE  2022-12-26T22:29:40Z    1               1               1                       c5.xlarge       AL2_x86_64      eks-esNodeGroup-f6c2a847-236b-033a-fd38-9733d80feddd managed
admin@ip-172-82-11-48:~/workspace/new-cluster$ eksctl get nodegroup --cluster escluster
CLUSTER         NODEGROUP       STATUS          CREATED                 MIN SIZE        MAX SIZE        DESIRED CAPACITY        INSTANCE TYPE   IMAGE ID                ASG NAME    TYPE
escluster       admin           CREATE_COMPLETE 2022-12-28T01:42:19Z    1               1               1                       m5.large        ami-0c9424a408e18bcc9   eksctl-escluster-nodegroup-admin-NodeGroup-O2KFDZEBC0CG              unmanaged
escluster       elastic         CREATE_COMPLETE 2022-12-28T01:42:19Z    3               3               3                       c5.2xlarge      ami-0c9424a408e18bcc9   eksctl-escluster-nodegroup-elastic-NodeGroup-NHNPBCTTS7DG    unmanaged

Failed to create volume

admin@ip-172-82-11-48:~/workspace/new-cluster$ kubectl describe pvc data-elasticsearch-data-0                                                                                        
Name:          data-elasticsearch-data-0                                                                                                                                             
Namespace:     default                                                                                                                                                               
StorageClass:  gp2                                                                                                                                                                   
Status:        Pending                                                                                                                                                               
Volume:                                                                                                                                                                              
Labels:        app.kubernetes.io/component=data                                                                                                                                      
               app.kubernetes.io/instance=elasticsearch                                                                                                                              
               app.kubernetes.io/name=elasticsearch                                                                                                                                  
Annotations:   volume.beta.kubernetes.io/storage-provisioner: ebs.csi.aws.com                                                                                                        
               volume.kubernetes.io/selected-node: ip-192-168-83-131.ec2.internal                                                                                                    
               volume.kubernetes.io/storage-provisioner: ebs.csi.aws.com                                                                                                             
Finalizers:    [kubernetes.io/pvc-protection]                                                                                                                                        
Capacity:                                                                                                                                                                            
Access Modes:                                                                                                                                                                        
VolumeMode:    Filesystem
Used By:       elasticsearch-data-0
Events:
  Type     Reason                Age                  From                                                                                      Message
  ----     ------                ----                 ----                                                                                      -------
  Normal   WaitForFirstConsumer  2m29s                persistentvolume-controller                                                               waiting for first consumer to be created before binding
  Warning  ProvisioningFailed    58s (x6 over 2m19s)  ebs.csi.aws.com_ebs-csi-controller-6f7bfdc97f-jtzbq_fbda4298-cf5f-4ea7-82aa-2d41b9083e1e  failed to provision volume with StorageClass "gp2": rpc error: code = DeadlineExceeded desc = context deadline exceeded
  Normal   Provisioning          26s (x7 over 2m29s)  ebs.csi.aws.com_ebs-csi-controller-6f7bfdc97f-jtzbq_fbda4298-cf5f-4ea7-82aa-2d41b9083e1e  External provisioner is provisioning volume for claim "default/data-elasticsearch-data-0"
  Warning  ProvisioningFailed    16s                  ebs.csi.aws.com_ebs-csi-controller-6f7bfdc97f-jtzbq_fbda4298-cf5f-4ea7-82aa-2d41b9083e1e  failed to provision volume with StorageClass "gp2": rpc error: code = Internal desc = Could not create volume "pvc-c61f5e44-257a-4326-8f64-8b0b4b447d63": could not create volume in EC2: RequestCanceled: request context canceled
caused by: context deadline exceeded
  Normal  ExternalProvisioning  6s (x12 over 2m29s)  persistentvolume-controller  waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator

Set es-gp2 as default csi

kubectl patch storageclass es-gp2 -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

calica

OhMyTiUP$ kubectl describe pod calico-node-r9xf6 -n kube-system
... ...
Events:
  Type     Reason     Age                     From     Message
  ----     ------     ----                    ----     -------
  Warning  Unhealthy  4m8s (x1010 over 152m)  kubelet  (combined from similar events): Readiness probe failed: 2022-12-28 05:59:12.666 [INFO][4287] confd/health.go 180: Number of node(s) with BGP peering established = 0
calico/node is not ready: BIRD is not ready: BGP not established with 192.168.84.163,192.168.91.125,192.168.94.53

Instances failed to join the kubernetes cluster(NodeCreationFailure)

The above occurred when tried to generate the node group using golang sdk

“Ignoring ingress because of error while validating ingress class” ingress=”default/helloworld” error=”no object matching key "nginx" in local store”

The curl is 404. Not found nginx

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: helloworld
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    kubernetes.io/ingress.class: "nginx"                       # Added this annotations
spec:
        #  ingressClassName: nginx                             # comment out
  defaultBackend:
    service:
      name: elasticsearch-master
      port:
        number: 9200
.. ...

Failed to create volume

failed to provision volume with StorageClass "es-gp2": rpc error: code = Internal desc = Could not create volume "pvc-3828d9df-e623-4ec2-8b8a-da36ef66f2e1": could not create volume in EC2: WebIdentityErr: failed to retrieve credentials
caused by: AccessDenied: Not authorized to perform sts:AssumeRoleWithWebIdentity

OpenID Connect provider URL -> associate OIDC to IAM -> Create service account in the k8s referring to iam role AWS storage

Failed to sink data to es

curl http://172.83.1.100:8083/connectors/ESSINK/status | jq 
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  2733  100  2733    0     0   266k      0 --:--:-- --:--:-- --:--:--  266k
{
  "name": "ESSINK",
  "connector": {
    "state": "RUNNING",
    "worker_id": "172.83.1.100:8083"
  },
  "tasks": [
    {
      "id": 0,
      "state": "FAILED",
      "worker_id": "172.83.1.100:8083",
      "trace": "org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:206)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132)\n\tat org.apache.kafka.connect.runtime.WorkerSinkTask.convertAndTransformRecord(WorkerSinkTask.java:513)\n\tat org.apache.kafka.connect.runtime.WorkerSinkTask.convertMessages(WorkerSinkTask.java:493)\n\tat org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:332)\n\tat org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:234)\n\tat org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:203)\n\tat org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:188)\n\tat org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:243)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: java.nio.BufferUnderflowException\n\tat java.base/java.nio.Buffer.nextGetIndex(Buffer.java:643)\n\tat java.base/java.nio.HeapByteBuffer.get(HeapByteBuffer.java:165)\n\tat io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe.getByteBuffer(AbstractKafkaSchemaSerDe.java:251)\n\tat io.confluent.kafka.serializers.AbstractKafkaAvroDeserializer$DeserializationContext.<init>(AbstractKafkaAvroDeserializer.java:334)\n\tat io.confluent.kafka.serializers.AbstractKafkaAvroDeserializer.deserializeWithSchemaAndVersion(AbstractKafkaAvroDeserializer.java:202)\n\tat io.confluent.connect.avro.AvroConverter$Deserializer.deserialize(AvroConverter.java:172)\n\tat io.confluent.connect.avro.AvroConverter.toConnectData(AvroConverter.java:107)\n\tat org.apache.kafka.connect.storage.Converter.toConnectData(Converter.java:87)\n\tat org.apache.kafka.connect.runtime.WorkerSinkTask.lambda$convertAndTransformRecord$3(WorkerSinkTask.java:513)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190)\n\t... 13 more\n"
    }
  ],
  "type": "sink"
}

The data format can not use canal-json which can not be converted to json.

Count the index

OhMyTiUP$ curl -u elastic:1234Abcd http://172.89.3.239/test_test01/_count | jq 
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100    71  100    71    0     0   4733      0 --:--:-- --:--:-- --:--:--  5071
{
  "count": 4,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  }
}

OhMyTiUP$  curl -u elastic:1234Abcd http://172.89.3.239/test_test01/_search?q=col01:5188146770730811393 | jq 
{
  "took": 28,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 1,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "test_test01",
        "_id": "test_test01+2+0",
        "_score": 1,
        "_source": {
          "col01": 5188146770730811000,
          "col02": 2
        }
      }
    ]
  }
}

Create the role instead of eksctl

cli$ permisson
AmazonEBSCSIDriverPolicy

cli$ trust relationships
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Federated": "arn:aws:iam::729581434105:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/32A13D8E01469C25DC2F92F1A2E42DC1"
            },
            "Action": "sts:AssumeRoleWithWebIdentity",
            "Condition": {
                "StringEquals": {
                    "oidc.eks.us-east-1.amazonaws.com/id/32A13D8E01469C25DC2F92F1A2E42DC1:aud": "sts.amazonaws.com",
                    "oidc.eks.us-east-1.amazonaws.com/id/32A13D8E01469C25DC2F92F1A2E42DC1:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa"
                }
            }
        }
    ]
}