2022/03/29 00:57:57 Etcd Client Initialization failed with error: Error creating etcd client: dial tcp: lookup multicloud-k8s-etcd on 169.254.25.10:53: no such host 2022/03/29 00:57:58 Starting Kubernetes Multicloud API 2022/03/29 00:57:58 Error Getting TLS Configuration. Starting without TLS... 10.233.68.25 - - [29/Mar/2022:02:35:26 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:02:37:01 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [29/Mar/2022:02:38:06 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.68.25 - - [29/Mar/2022:02:38:29 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:02:39:00 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:02:40:19 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:02:41:43 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [29/Mar/2022:02:42:29 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.68.25 - - [29/Mar/2022:02:43:06 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [29/Mar/2022:02:44:26 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.68.25 - - [29/Mar/2022:02:44:58 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:02:47:43 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:02:50:26 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [29/Mar/2022:02:54:26 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [29/Mar/2022:03:08:54 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 10.233.68.25 - - [29/Mar/2022:03:27:40 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.68.25 - - [29/Mar/2022:03:27:40 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7665 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "POST /v1/rb/definition HTTP/1.1" 201 124 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "GET /v1/rb/definition/cds-ms-rb-23091ff1-13a7-486d-bbf0-11ec836316f6/v1 HTTP/1.1" 200 124 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "POST /v1/rb/definition/cds-ms-rb-23091ff1-13a7-486d-bbf0-11ec836316f6/v1/content HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "GET /v1/rb/definition/cds-ms-rb-23091ff1-13a7-486d-bbf0-11ec836316f6/v1 HTTP/1.1" 200 138 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "POST /v1/rb/definition/cds-ms-rb-23091ff1-13a7-486d-bbf0-11ec836316f6/v1/profile HTTP/1.1" 201 305 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "GET /v1/rb/definition/cds-ms-rb-23091ff1-13a7-486d-bbf0-11ec836316f6/v1/profile/cds-ms-prof-01436b49-ca4f-4d23-8032-348836db8373 HTTP/1.1" 200 305 10.233.68.25 - - [29/Mar/2022:03:27:41 +0000] "POST /v1/rb/definition/cds-ms-rb-23091ff1-13a7-486d-bbf0-11ec836316f6/v1/profile/cds-ms-prof-01436b49-ca4f-4d23-8032-348836db8373/content HTTP/1.1" 200 0 2022/03/29 03:27:42 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/29 03:27:42 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/29 03:27:42 Main rss info 2022/03/29 03:27:42 Path: /tmp/helm-tmpl-630718463/manifest-0 2022/03/29 03:27:42 Kind: Service 2022/03/29 03:27:42 Path: /tmp/helm-tmpl-630718463/manifest-1 2022/03/29 03:27:42 Kind: Deployment 2022/03/29 03:27:42 Crd rss info 2022/03/29 03:27:42 Hook info {"level":"info","msg":"Creating Namespace","namespace":"onap-tests","time":"2022-03-29T03:27:42Z"} 2022/03/29 03:27:42 Namespace (onap-tests) created {"filepath":"/tmp/helm-tmpl-630718463/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:27:42Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-regression-mockserver","time":"2022-03-29T03:27:42Z"} {"filepath":"/tmp/helm-tmpl-630718463/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:27:42Z"} 2022/03/29 03:27:42 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-cds-mockserver","time":"2022-03-29T03:27:42Z"} 10.233.68.25 - - [29/Mar/2022:03:27:42 +0000] "POST /v1/instance HTTP/1.1" 201 535 2022/03/29 03:28:58 [Config Cleanup] Instance eloquent_wu 2022/03/29 03:28:58 [scheduleResources]: START thread {"error":"Cleanup Config Resources: Retrieving active config list info: Get Config DB Entry: Get Config DB List: Etcd Client not initialized","level":"error","msg":"Error Deleting Instance","time":"2022-03-29T03:29:03Z"} 10.233.68.25 - - [29/Mar/2022:03:28:58 +0000] "DELETE /v1/instance/eloquent_wu HTTP/1.1" 500 131 127.0.0.1 - - [29/Mar/2022:03:29:10 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "GET /v1/rb/definition/pnf-cnf-rb-8a0c6c23-5e1f-4711-ad2f-d18176363ff9/v1 HTTP/1.1" 200 125 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "POST /v1/rb/definition/pnf-cnf-rb-8a0c6c23-5e1f-4711-ad2f-d18176363ff9/v1/content HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "GET /v1/rb/definition/pnf-cnf-rb-8a0c6c23-5e1f-4711-ad2f-d18176363ff9/v1 HTTP/1.1" 200 138 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "POST /v1/rb/definition/pnf-cnf-rb-8a0c6c23-5e1f-4711-ad2f-d18176363ff9/v1/profile HTTP/1.1" 201 314 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "GET /v1/rb/definition/pnf-cnf-rb-8a0c6c23-5e1f-4711-ad2f-d18176363ff9/v1/profile/pnf-cnf-profile-2c01e47b-aa5c-4475-9441-077efd72d26e HTTP/1.1" 200 314 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "POST /v1/rb/definition/pnf-cnf-rb-8a0c6c23-5e1f-4711-ad2f-d18176363ff9/v1/profile/pnf-cnf-profile-2c01e47b-aa5c-4475-9441-077efd72d26e/content HTTP/1.1" 200 0 2022/03/29 03:32:14 Main rss info 2022/03/29 03:32:14 Path: /tmp/helm-tmpl-328935598/manifest-0 2022/03/29 03:32:14 Kind: Service 2022/03/29 03:32:14 Path: /tmp/helm-tmpl-328935598/manifest-1 2022/03/29 03:32:14 Kind: Service 2022/03/29 03:32:14 Path: /tmp/helm-tmpl-328935598/manifest-2 2022/03/29 03:32:14 Kind: Pod 2022/03/29 03:32:14 Path: /tmp/helm-tmpl-328935598/manifest-3 2022/03/29 03:32:14 Kind: Deployment 2022/03/29 03:32:14 Crd rss info 2022/03/29 03:32:14 Hook info {"filepath":"/tmp/helm-tmpl-328935598/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:32:14Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-29T03:32:14Z"} {"filepath":"/tmp/helm-tmpl-328935598/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:32:14Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-29T03:32:14Z"} {"filepath":"/tmp/helm-tmpl-328935598/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:32:14Z"} 2022/03/29 03:32:14 No plugin for kind Pod found. Using generic Plugin 2022/03/29 03:32:14 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-29T03:32:14Z"} {"filepath":"/tmp/helm-tmpl-328935598/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:32:14Z"} 2022/03/29 03:32:14 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-29T03:32:14Z"} 10.233.68.25 - - [29/Mar/2022:03:32:14 +0000] "POST /v1/instance HTTP/1.1" 201 731 10.233.68.25 - - [29/Mar/2022:03:34:37 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 10.233.68.25 - - [29/Mar/2022:03:34:37 +0000] "POST /v1/rb/definition HTTP/1.1" 201 124 10.233.68.25 - - [29/Mar/2022:03:34:37 +0000] "GET /v1/rb/definition/cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a/v1 HTTP/1.1" 200 124 10.233.68.25 - - [29/Mar/2022:03:34:37 +0000] "POST /v1/rb/definition/cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a/v1/content HTTP/1.1" 200 0 10.233.68.25 - - [29/Mar/2022:03:34:37 +0000] "GET /v1/rb/definition/cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a/v1 HTTP/1.1" 200 138 10.233.68.25 - - [29/Mar/2022:03:34:38 +0000] "POST /v1/rb/definition/cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a/v1/profile HTTP/1.1" 201 305 10.233.68.25 - - [29/Mar/2022:03:34:38 +0000] "GET /v1/rb/definition/cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a/v1/profile/cds-ms-prof-24643a5b-a874-4bea-a20f-141bd653288a HTTP/1.1" 200 305 10.233.68.25 - - [29/Mar/2022:03:34:38 +0000] "POST /v1/rb/definition/cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a/v1/profile/cds-ms-prof-24643a5b-a874-4bea-a20f-141bd653288a/content HTTP/1.1" 200 0 2022/03/29 03:34:38 Main rss info 2022/03/29 03:34:38 Path: /tmp/helm-tmpl-845697593/manifest-0 2022/03/29 03:34:38 Kind: Service 2022/03/29 03:34:38 Path: /tmp/helm-tmpl-845697593/manifest-1 2022/03/29 03:34:38 Kind: Deployment 2022/03/29 03:34:38 Crd rss info 2022/03/29 03:34:38 Hook info {"filepath":"/tmp/helm-tmpl-845697593/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-29T03:34:38Z"} {"error":"Create Service error: Service \"cds-regression-mockserver\" is invalid: spec.ports[0].nodePort: Invalid value: 30726: provided port is already allocated","filepath":"/tmp/helm-tmpl-845697593/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-29T03:34:38Z"} 2022/03/29 03:34:38 Instance: wizardly_feistel, Main rss are failed, skip post-install and remove instance in DB 2022/03/29 03:34:38 Deleted Obj with ID ObjectID("62427e4ed5734542f96bcba5") 2022/03/29 03:34:38 Deleted Obj with ID ObjectID("62427e4ee7f96268a2cca912") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: Service \"cds-regression-mockserver\" is invalid: spec.ports[0].nodePort: Invalid value: 30726: provided port is already allocated","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"cds-ms-rb-43f8c2b1-e243-4c50-bfe6-02aef5ed643a","rb-version":"v1","profile-name":"cds-ms-prof-24643a5b-a874-4bea-a20f-141bd653288a","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-29T03:34:38Z"} 10.233.68.25 - - [29/Mar/2022:03:34:38 +0000] "POST /v1/instance HTTP/1.1" 500 252 10.233.68.25 - - [29/Mar/2022:03:34:38 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 2022/03/29 03:34:38 Deleted Obj with ID ObjectID("62427cacd5734542f96bcb89") 10.233.68.25 - - [29/Mar/2022:03:34:38 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/29 03:34:38 Deleted Obj with ID ObjectID("62427cade7f96268a2cca739") 10.233.68.25 - - [29/Mar/2022:03:40:43 +0000] "DELETE /v1/instance/kind_carson HTTP/1.1" 500 177 {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-29T03:40:43Z"}