2022/03/28 02:43:13 Etcd Client Initialization failed with error: Error creating etcd client: dial tcp: lookup multicloud-k8s-etcd on 169.254.25.10:53: no such host 2022/03/28 02:43:13 Starting Kubernetes Multicloud API 2022/03/28 02:43:13 Error Getting TLS Configuration. Starting without TLS... 10.233.65.184 - - [28/Mar/2022:03:49:29 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [28/Mar/2022:03:52:19 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [28/Mar/2022:03:52:32 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "GET /v1/rb/definition/pnf-cnf-rb-8a332e51-5a77-4643-84a0-58448f064a9c/v1 HTTP/1.1" 200 125 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "POST /v1/rb/definition/pnf-cnf-rb-8a332e51-5a77-4643-84a0-58448f064a9c/v1/content HTTP/1.1" 200 0 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "GET /v1/rb/definition/pnf-cnf-rb-8a332e51-5a77-4643-84a0-58448f064a9c/v1 HTTP/1.1" 200 138 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "POST /v1/rb/definition/pnf-cnf-rb-8a332e51-5a77-4643-84a0-58448f064a9c/v1/profile HTTP/1.1" 201 308 10.233.65.184 - - [28/Mar/2022:04:04:07 +0000] "GET /v1/rb/definition/pnf-cnf-rb-8a332e51-5a77-4643-84a0-58448f064a9c/v1/profile/pnf-cnf-profile-ed33b95a-a285-42b8-b8c9-3cca7280cc97 HTTP/1.1" 200 308 10.233.65.184 - - [28/Mar/2022:04:04:08 +0000] "POST /v1/rb/definition/pnf-cnf-rb-8a332e51-5a77-4643-84a0-58448f064a9c/v1/profile/pnf-cnf-profile-ed33b95a-a285-42b8-b8c9-3cca7280cc97/content HTTP/1.1" 200 0 2022/03/28 04:04:08 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/28 04:04:08 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/28 04:04:08 Main rss info 2022/03/28 04:04:08 Path: /tmp/helm-tmpl-914565338/manifest-0 2022/03/28 04:04:08 Kind: Service 2022/03/28 04:04:08 Path: /tmp/helm-tmpl-914565338/manifest-1 2022/03/28 04:04:08 Kind: Service 2022/03/28 04:04:08 Path: /tmp/helm-tmpl-914565338/manifest-2 2022/03/28 04:04:08 Kind: Pod 2022/03/28 04:04:08 Path: /tmp/helm-tmpl-914565338/manifest-3 2022/03/28 04:04:08 Kind: Deployment 2022/03/28 04:04:08 Crd rss info 2022/03/28 04:04:08 Hook info {"filepath":"/tmp/helm-tmpl-914565338/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:04:08Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-28T04:04:08Z"} {"filepath":"/tmp/helm-tmpl-914565338/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:04:08Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-28T04:04:08Z"} {"filepath":"/tmp/helm-tmpl-914565338/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:04:08Z"} 2022/03/28 04:04:08 No plugin for kind Pod found. Using generic Plugin 2022/03/28 04:04:08 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-28T04:04:08Z"} {"filepath":"/tmp/helm-tmpl-914565338/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:04:08Z"} 2022/03/28 04:04:08 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-28T04:04:08Z"} 10.233.65.184 - - [28/Mar/2022:04:04:08 +0000] "POST /v1/instance HTTP/1.1" 201 723 10.233.65.184 - - [28/Mar/2022:04:06:56 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 127.0.0.1 - - [28/Mar/2022:04:07:08 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [28/Mar/2022:04:07:08 +0000] "POST /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/content HTTP/1.1" 200 0 127.0.0.1 - - [28/Mar/2022:04:07:23 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [28/Mar/2022:04:07:39 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 127.0.0.1 - - [28/Mar/2022:04:08:00 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.65.184 - - [28/Mar/2022:04:11:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "GET /v1/rb/definition/pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e/v1 HTTP/1.1" 200 125 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "POST /v1/rb/definition/pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e/v1/content HTTP/1.1" 200 0 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "GET /v1/rb/definition/pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e/v1 HTTP/1.1" 200 138 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "POST /v1/rb/definition/pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e/v1/profile HTTP/1.1" 201 308 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "GET /v1/rb/definition/pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e/v1/profile/pnf-cnf-profile-a0cf778d-4d50-408e-8eb7-57b421f677f6 HTTP/1.1" 200 308 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "POST /v1/rb/definition/pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e/v1/profile/pnf-cnf-profile-a0cf778d-4d50-408e-8eb7-57b421f677f6/content HTTP/1.1" 200 0 2022/03/28 04:11:15 Main rss info 2022/03/28 04:11:15 Path: /tmp/helm-tmpl-403049373/manifest-0 2022/03/28 04:11:15 Kind: Service 2022/03/28 04:11:15 Path: /tmp/helm-tmpl-403049373/manifest-1 2022/03/28 04:11:15 Kind: Service 2022/03/28 04:11:15 Path: /tmp/helm-tmpl-403049373/manifest-2 2022/03/28 04:11:15 Kind: Pod 2022/03/28 04:11:15 Path: /tmp/helm-tmpl-403049373/manifest-3 2022/03/28 04:11:15 Kind: Deployment 2022/03/28 04:11:15 Crd rss info 2022/03/28 04:11:15 Hook info {"filepath":"/tmp/helm-tmpl-403049373/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:11:15Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-403049373/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-28T04:11:15Z"} 2022/03/28 04:11:15 Instance: happy_ritchie, Main rss are failed, skip post-install and remove instance in DB 2022/03/28 04:11:15 Deleted Obj with ID ObjectID("62413563b6b957771ce2c3f5") 2022/03/28 04:11:15 Deleted Obj with ID ObjectID("624135632300ed65f2f1a6d5") 10.233.65.184 - - [28/Mar/2022:04:11:15 +0000] "POST /v1/instance HTTP/1.1" 500 180 {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-e27e2834-03f4-4fd8-95bf-e02cdce91d1e","rb-version":"v1","profile-name":"pnf-cnf-profile-a0cf778d-4d50-408e-8eb7-57b421f677f6","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-28T04:11:15Z"} 10.233.65.184 - - [28/Mar/2022:04:13:46 +0000] "GET /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05 HTTP/1.1" 200 270 10.233.65.184 - - [28/Mar/2022:04:13:46 +0000] "GET /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/profile/cnftest HTTP/1.1" 404 87 10.233.65.184 - - [28/Mar/2022:04:13:46 +0000] "POST /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/profile HTTP/1.1" 201 240 10.233.65.184 - - [28/Mar/2022:04:13:46 +0000] "GET /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/profile/cnftest HTTP/1.1" 200 240 10.233.65.184 - - [28/Mar/2022:04:13:46 +0000] "POST /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/profile/cnftest/content HTTP/1.1" 200 0 10.233.65.184 - - [28/Mar/2022:04:14:02 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-9111c220-0017-44c5-b077-588f3ed567d1 HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-28T04:14:02Z"} {"Body":{},"Payload":{"generic-vnf-id":"1549b25b-cb30-4fca-a698-55ae051959ef","vf-module-id":"3e8d3a5e-e164-4f36-b814-97e0c8ce6bef","vf-module-model-invariant-id":"f58a2c7e-edd0-46f9-8964-e5711f68729d","vf-module-model-version-id":"5a716a02-d8ff-4964-b307-ed3ad7205ac6","vf-module-model-customization-id":"02fd08ef-1e0e-4305-bf8f-f70d68be9b05","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-9111c220-0017-44c5-b077-588f3ed567d1"}},"level":"info","msg":"Broker API Payload","time":"2022-03-28T04:14:03Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"f58a2c7e-edd0-46f9-8964-e5711f68729d","rb-version":"02fd08ef-1e0e-4305-bf8f-f70d68be9b05","profile-name":"cnftest","release-name":"3e8d3a5e-e164-4f36-b814-97e0c8ce6bef","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-9111c220-0017-44c5-b077-588f3ed567d1"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-28T04:14:03Z"} 2022/03/28 04:14:03 Main rss info 2022/03/28 04:14:03 Path: /tmp/helm-tmpl-012008481/manifest-0 2022/03/28 04:14:03 Kind: Service 2022/03/28 04:14:03 Path: /tmp/helm-tmpl-012008481/manifest-1 2022/03/28 04:14:03 Kind: Deployment 2022/03/28 04:14:03 Crd rss info 2022/03/28 04:14:03 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-28T04:14:03Z"} 2022/03/28 04:14:03 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-012008481/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:14:03Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-28T04:14:03Z"} {"filepath":"/tmp/helm-tmpl-012008481/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-28T04:14:03Z"} 2022/03/28 04:14:03 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-28T04:14:03Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_pascal","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-28T04:14:03Z"} 10.233.65.184 - - [28/Mar/2022:04:14:03 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 305 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_pascal","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-28T04:14:03Z"} 10.233.65.184 - - [28/Mar/2022:04:14:03 +0000] "GET /basicnf-owner/k8sregion/infra_workload/stupefied_pascal HTTP/1.1" 200 124 10.233.65.184 - - [28/Mar/2022:04:14:04 +0000] "POST /basicnf-owner/k8sregion/infra_workload/stupefied_pascal HTTP/1.1" 405 0 10.233.65.184 - - [28/Mar/2022:04:14:32 +0000] "GET /basicnf-owner/k8sregion/infra_workload/stupefied_pascal HTTP/1.1" 200 124 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_pascal","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-28T04:14:32Z"} 2022/03/28 04:14:33 [Config Cleanup] Instance stupefied_pascal 2022/03/28 04:14:33 [scheduleResources]: START thread 10.233.65.184 - - [28/Mar/2022:04:14:33 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/stupefied_pascal HTTP/1.1" 500 131 10.233.65.184 - - [28/Mar/2022:04:14:49 +0000] "GET /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05 HTTP/1.1" 200 270 10.233.65.184 - - [28/Mar/2022:04:14:49 +0000] "GET /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/profile/cnftest HTTP/1.1" 200 240 2022/03/28 04:14:49 Deleted Obj with ID ObjectID("624135fab6b957771ce2c3f6") 2022/03/28 04:14:49 Deleted Obj with ID ObjectID("624135fab6b957771ce2c3f7") 10.233.65.184 - - [28/Mar/2022:04:14:49 +0000] "DELETE /v1/rb/definition/f58a2c7e-edd0-46f9-8964-e5711f68729d/02fd08ef-1e0e-4305-bf8f-f70d68be9b05/profile/cnftest HTTP/1.1" 204 0 2022/03/28 04:14:49 Deleted Obj with ID ObjectID("624135f92300ed65f2f1a742") 10.233.65.184 - - [28/Mar/2022:04:15:26 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 2022/03/28 04:15:26 Deleted Obj with ID ObjectID("624133b7b6b957771ce2c3d6") 10.233.65.184 - - [28/Mar/2022:04:15:26 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/28 04:15:26 Deleted Obj with ID ObjectID("624133b72300ed65f2f1a511")