2022/03/23 01:49:50 Etcd Client Initialization failed with error: Error creating etcd client: dial tcp: lookup multicloud-k8s-etcd on 169.254.25.10:53: no such host 2022/03/23 01:49:52 Starting Kubernetes Multicloud API 2022/03/23 01:49:52 Error Getting TLS Configuration. Starting without TLS... 10.233.67.3 - - [23/Mar/2022:03:20:25 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [23/Mar/2022:03:26:12 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [23/Mar/2022:03:36:31 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [23/Mar/2022:03:50:31 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 10.233.67.3 - - [23/Mar/2022:03:59:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.67.3 - - [23/Mar/2022:03:59:14 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7565 10.233.67.3 - - [23/Mar/2022:03:59:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 127.0.0.1 - - [23/Mar/2022:03:59:34 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [23/Mar/2022:03:59:34 +0000] "POST /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/content HTTP/1.1" 200 0 10.233.67.3 - - [23/Mar/2022:04:02:44 +0000] "GET /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9 HTTP/1.1" 200 270 10.233.67.3 - - [23/Mar/2022:04:02:44 +0000] "GET /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/profile/cnftest HTTP/1.1" 404 87 10.233.67.3 - - [23/Mar/2022:04:02:44 +0000] "POST /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/profile HTTP/1.1" 201 240 10.233.67.3 - - [23/Mar/2022:04:02:44 +0000] "GET /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/profile/cnftest HTTP/1.1" 200 240 10.233.67.3 - - [23/Mar/2022:04:02:44 +0000] "POST /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/profile/cnftest/content HTTP/1.1" 200 0 10.233.67.3 - - [23/Mar/2022:04:02:52 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-da813383-796a-40e0-92a6-42a5f6eaf01f HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-23T04:02:52Z"} {"Body":{},"Payload":{"generic-vnf-id":"3d7ead96-d8ad-4a8e-b919-6a389fc520c1","vf-module-id":"bea69260-542d-44c9-a914-ac18b225f4a3","vf-module-model-invariant-id":"fa15af11-4ab0-404c-a247-d7d089ef4c73","vf-module-model-version-id":"a6ed7fb3-fd86-46fa-b958-f5977d482b0c","vf-module-model-customization-id":"94816983-9008-4c2d-af82-0b5e89af86d9","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-da813383-796a-40e0-92a6-42a5f6eaf01f"}},"level":"info","msg":"Broker API Payload","time":"2022-03-23T04:02:52Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"fa15af11-4ab0-404c-a247-d7d089ef4c73","rb-version":"94816983-9008-4c2d-af82-0b5e89af86d9","profile-name":"cnftest","release-name":"bea69260-542d-44c9-a914-ac18b225f4a3","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-da813383-796a-40e0-92a6-42a5f6eaf01f"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-23T04:02:52Z"} 2022/03/23 04:02:52 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/23 04:02:52 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/23 04:02:52 Main rss info 2022/03/23 04:02:52 Path: /tmp/helm-tmpl-617517686/manifest-0 2022/03/23 04:02:52 Kind: Service 2022/03/23 04:02:52 Path: /tmp/helm-tmpl-617517686/manifest-1 2022/03/23 04:02:52 Kind: Deployment 2022/03/23 04:02:52 Crd rss info 2022/03/23 04:02:52 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-23T04:02:52Z"} 2022/03/23 04:02:52 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-617517686/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:02:52Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-23T04:02:52Z"} {"filepath":"/tmp/helm-tmpl-617517686/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:02:52Z"} 2022/03/23 04:02:52 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-23T04:02:52Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"angry_chatelet","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:02:52Z"} 10.233.67.3 - - [23/Mar/2022:04:02:52 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 303 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"angry_chatelet","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:02:52Z"} 10.233.67.3 - - [23/Mar/2022:04:02:52 +0000] "GET /basicnf-owner/k8sregion/infra_workload/angry_chatelet HTTP/1.1" 200 122 10.233.67.3 - - [23/Mar/2022:04:02:52 +0000] "POST /basicnf-owner/k8sregion/infra_workload/angry_chatelet HTTP/1.1" 405 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"angry_chatelet","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:03:23Z"} 10.233.67.3 - - [23/Mar/2022:04:03:23 +0000] "GET /basicnf-owner/k8sregion/infra_workload/angry_chatelet HTTP/1.1" 200 122 2022/03/23 04:03:23 [Config Cleanup] Instance angry_chatelet 2022/03/23 04:03:23 [scheduleResources]: START thread 10.233.67.3 - - [23/Mar/2022:04:03:23 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/angry_chatelet HTTP/1.1" 500 131 10.233.67.3 - - [23/Mar/2022:04:03:42 +0000] "GET /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9 HTTP/1.1" 200 270 10.233.67.3 - - [23/Mar/2022:04:03:42 +0000] "GET /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/profile/cnftest HTTP/1.1" 200 240 2022/03/23 04:03:42 Deleted Obj with ID ObjectID("623a9be47000b7b8c3b374b2") 2022/03/23 04:03:42 Deleted Obj with ID ObjectID("623a9be47000b7b8c3b374b3") 10.233.67.3 - - [23/Mar/2022:04:03:42 +0000] "DELETE /v1/rb/definition/fa15af11-4ab0-404c-a247-d7d089ef4c73/94816983-9008-4c2d-af82-0b5e89af86d9/profile/cnftest HTTP/1.1" 204 0 2022/03/23 04:03:42 Deleted Obj with ID ObjectID("623a9be423d406f98341dfd5") 10.233.67.3 - - [23/Mar/2022:04:04:10 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 2022/03/23 04:04:10 Deleted Obj with ID ObjectID("623a9b127000b7b8c3b374ac") 10.233.67.3 - - [23/Mar/2022:04:04:10 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/23 04:04:10 Deleted Obj with ID ObjectID("623a9b1223d406f98341df35") 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7565 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "GET /v1/rb/definition/pnf-cnf-rb-7ab41cdf-77fe-4587-9bcf-e377da38c6c4/v1 HTTP/1.1" 200 125 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "POST /v1/rb/definition/pnf-cnf-rb-7ab41cdf-77fe-4587-9bcf-e377da38c6c4/v1/content HTTP/1.1" 200 0 10.233.67.3 - - [23/Mar/2022:04:12:10 +0000] "GET /v1/rb/definition/pnf-cnf-rb-7ab41cdf-77fe-4587-9bcf-e377da38c6c4/v1 HTTP/1.1" 200 138 10.233.67.3 - - [23/Mar/2022:04:12:11 +0000] "POST /v1/rb/definition/pnf-cnf-rb-7ab41cdf-77fe-4587-9bcf-e377da38c6c4/v1/profile HTTP/1.1" 201 308 10.233.67.3 - - [23/Mar/2022:04:12:11 +0000] "GET /v1/rb/definition/pnf-cnf-rb-7ab41cdf-77fe-4587-9bcf-e377da38c6c4/v1/profile/pnf-cnf-profile-c7f263c8-2855-4288-bc54-9a14c9b00f4a HTTP/1.1" 200 308 10.233.67.3 - - [23/Mar/2022:04:12:11 +0000] "POST /v1/rb/definition/pnf-cnf-rb-7ab41cdf-77fe-4587-9bcf-e377da38c6c4/v1/profile/pnf-cnf-profile-c7f263c8-2855-4288-bc54-9a14c9b00f4a/content HTTP/1.1" 200 0 2022/03/23 04:12:11 Main rss info 2022/03/23 04:12:11 Path: /tmp/helm-tmpl-088353766/manifest-0 2022/03/23 04:12:11 Kind: Service 2022/03/23 04:12:11 Path: /tmp/helm-tmpl-088353766/manifest-1 2022/03/23 04:12:11 Kind: Service 2022/03/23 04:12:11 Path: /tmp/helm-tmpl-088353766/manifest-2 2022/03/23 04:12:11 Kind: Pod 2022/03/23 04:12:11 Path: /tmp/helm-tmpl-088353766/manifest-3 2022/03/23 04:12:11 Kind: Deployment 2022/03/23 04:12:11 Crd rss info 2022/03/23 04:12:11 Hook info {"filepath":"/tmp/helm-tmpl-088353766/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:12:11Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-23T04:12:11Z"} {"filepath":"/tmp/helm-tmpl-088353766/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:12:11Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-23T04:12:11Z"} {"filepath":"/tmp/helm-tmpl-088353766/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:12:11Z"} 2022/03/23 04:12:11 No plugin for kind Pod found. Using generic Plugin 2022/03/23 04:12:11 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-23T04:12:11Z"} {"filepath":"/tmp/helm-tmpl-088353766/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:12:11Z"} 2022/03/23 04:12:11 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-23T04:12:11Z"} 10.233.67.3 - - [23/Mar/2022:04:12:11 +0000] "POST /v1/instance HTTP/1.1" 201 729 127.0.0.1 - - [23/Mar/2022:04:17:37 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [23/Mar/2022:04:17:52 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 2022/03/23 04:19:29 [Config Cleanup] Instance nostalgic_hertz 2022/03/23 04:19:29 [scheduleResources]: START thread {"error":"Cleanup Config Resources: Retrieving active config list info: Get Config DB Entry: Get Config DB List: Etcd Client not initialized","level":"error","msg":"Error Deleting Instance","time":"2022-03-23T04:19:34Z"} 10.233.67.3 - - [23/Mar/2022:04:19:29 +0000] "DELETE /v1/instance/nostalgic_hertz HTTP/1.1" 500 131 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b/v1 HTTP/1.1" 200 125 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b/v1/content HTTP/1.1" 200 0 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b/v1 HTTP/1.1" 200 138 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b/v1/profile HTTP/1.1" 201 308 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b/v1/profile/pnf-cnf-profile-8988efa7-a68a-4c7c-8c24-fd737d626eea HTTP/1.1" 200 308 10.233.67.3 - - [23/Mar/2022:04:24:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b/v1/profile/pnf-cnf-profile-8988efa7-a68a-4c7c-8c24-fd737d626eea/content HTTP/1.1" 200 0 2022/03/23 04:24:59 Main rss info 2022/03/23 04:24:59 Path: /tmp/helm-tmpl-611420092/manifest-0 2022/03/23 04:24:59 Kind: Service 2022/03/23 04:24:59 Path: /tmp/helm-tmpl-611420092/manifest-1 2022/03/23 04:24:59 Kind: Service 2022/03/23 04:24:59 Path: /tmp/helm-tmpl-611420092/manifest-2 2022/03/23 04:24:59 Kind: Pod 2022/03/23 04:24:59 Path: /tmp/helm-tmpl-611420092/manifest-3 2022/03/23 04:24:59 Kind: Deployment 2022/03/23 04:24:59 Crd rss info 2022/03/23 04:24:59 Hook info {"filepath":"/tmp/helm-tmpl-611420092/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:24:59Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-611420092/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-23T04:24:59Z"} 2022/03/23 04:24:59 Instance: optimistic_sanderson, Main rss are failed, skip post-install and remove instance in DB 2022/03/23 04:24:59 Deleted Obj with ID ObjectID("623aa11b7000b7b8c3b374cc") 2022/03/23 04:24:59 Deleted Obj with ID ObjectID("623aa11b23d406f98341e3ef") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-319f56fa-1f6a-43c6-bc6a-91622dbdb39b","rb-version":"v1","profile-name":"pnf-cnf-profile-8988efa7-a68a-4c7c-8c24-fd737d626eea","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-23T04:24:59Z"} 10.233.67.3 - - [23/Mar/2022:04:24:59 +0000] "POST /v1/instance HTTP/1.1" 500 180