2022/03/10 14:26:54 Starting Kubernetes Multicloud API 2022/03/10 14:26:54 Error Getting TLS Configuration. Starting without TLS... 10.233.71.221 - - [10/Mar/2022:15:26:48 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [10/Mar/2022:15:30:19 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [10/Mar/2022:15:30:56 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [10/Mar/2022:15:40:55 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [10/Mar/2022:15:43:57 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "GET /v1/rb/definition/pnf-cnf-rb-062dd371-6eea-45ea-aac2-2a2e546f0913/v1 HTTP/1.1" 200 125 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "POST /v1/rb/definition/pnf-cnf-rb-062dd371-6eea-45ea-aac2-2a2e546f0913/v1/content HTTP/1.1" 200 0 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "GET /v1/rb/definition/pnf-cnf-rb-062dd371-6eea-45ea-aac2-2a2e546f0913/v1 HTTP/1.1" 200 138 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "POST /v1/rb/definition/pnf-cnf-rb-062dd371-6eea-45ea-aac2-2a2e546f0913/v1/profile HTTP/1.1" 201 308 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "GET /v1/rb/definition/pnf-cnf-rb-062dd371-6eea-45ea-aac2-2a2e546f0913/v1/profile/pnf-cnf-profile-79f0ebd8-b894-486c-8aa5-f330e93e9379 HTTP/1.1" 200 308 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "POST /v1/rb/definition/pnf-cnf-rb-062dd371-6eea-45ea-aac2-2a2e546f0913/v1/profile/pnf-cnf-profile-79f0ebd8-b894-486c-8aa5-f330e93e9379/content HTTP/1.1" 200 0 2022/03/10 15:51:30 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/10 15:51:30 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/10 15:51:30 Main rss info 2022/03/10 15:51:30 Path: /tmp/helm-tmpl-283295846/manifest-0 2022/03/10 15:51:30 Kind: Service 2022/03/10 15:51:30 Path: /tmp/helm-tmpl-283295846/manifest-1 2022/03/10 15:51:30 Kind: Service 2022/03/10 15:51:30 Path: /tmp/helm-tmpl-283295846/manifest-2 2022/03/10 15:51:30 Kind: Pod 2022/03/10 15:51:30 Path: /tmp/helm-tmpl-283295846/manifest-3 2022/03/10 15:51:30 Kind: Deployment 2022/03/10 15:51:30 Crd rss info 2022/03/10 15:51:30 Hook info {"filepath":"/tmp/helm-tmpl-283295846/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T15:51:31Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-10T15:51:31Z"} {"filepath":"/tmp/helm-tmpl-283295846/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T15:51:31Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-10T15:51:31Z"} {"filepath":"/tmp/helm-tmpl-283295846/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T15:51:31Z"} 2022/03/10 15:51:31 No plugin for kind Pod found. Using generic Plugin 2022/03/10 15:51:31 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-10T15:51:31Z"} {"filepath":"/tmp/helm-tmpl-283295846/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T15:51:31Z"} 2022/03/10 15:51:31 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-10T15:51:31Z"} 10.233.71.221 - - [10/Mar/2022:15:51:30 +0000] "POST /v1/instance HTTP/1.1" 201 731 10.233.71.221 - - [10/Mar/2022:15:55:28 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 127.0.0.1 - - [10/Mar/2022:15:55:41 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [10/Mar/2022:15:55:41 +0000] "POST /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/content HTTP/1.1" 200 0 127.0.0.1 - - [10/Mar/2022:15:55:54 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 127.0.0.1 - - [10/Mar/2022:15:56:14 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [10/Mar/2022:15:56:34 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.71.221 - - [10/Mar/2022:16:00:39 +0000] "GET /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80 HTTP/1.1" 200 270 10.233.71.221 - - [10/Mar/2022:16:00:39 +0000] "GET /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/profile/cnftest HTTP/1.1" 404 87 10.233.71.221 - - [10/Mar/2022:16:00:39 +0000] "POST /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/profile HTTP/1.1" 201 240 10.233.71.221 - - [10/Mar/2022:16:00:39 +0000] "GET /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/profile/cnftest HTTP/1.1" 200 240 10.233.71.221 - - [10/Mar/2022:16:00:40 +0000] "POST /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/profile/cnftest/content HTTP/1.1" 200 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-10T16:00:59Z"} 10.233.71.221 - - [10/Mar/2022:16:00:59 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-8386e2f1-7d62-4179-9acf-dcdcbdfdd637 HTTP/1.1" 200 114 {"Body":{},"Payload":{"generic-vnf-id":"e8ddbd6b-39a3-4d39-8ce1-7baf0fdeeb3d","vf-module-id":"53b2774e-59e4-4140-bb23-0aec8a7670d6","vf-module-model-invariant-id":"16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c","vf-module-model-version-id":"bccaa722-6004-423f-b14f-423cf9dd9c44","vf-module-model-customization-id":"37b9be22-cb7b-40e0-93ac-4626529e4a80","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-8386e2f1-7d62-4179-9acf-dcdcbdfdd637"}},"level":"info","msg":"Broker API Payload","time":"2022-03-10T16:01:01Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c","rb-version":"37b9be22-cb7b-40e0-93ac-4626529e4a80","profile-name":"cnftest","release-name":"53b2774e-59e4-4140-bb23-0aec8a7670d6","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-8386e2f1-7d62-4179-9acf-dcdcbdfdd637"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-10T16:01:01Z"} 2022/03/10 16:01:01 Main rss info 2022/03/10 16:01:01 Path: /tmp/helm-tmpl-631431042/manifest-0 2022/03/10 16:01:01 Kind: Service 2022/03/10 16:01:01 Path: /tmp/helm-tmpl-631431042/manifest-1 2022/03/10 16:01:01 Kind: Deployment 2022/03/10 16:01:01 Crd rss info 2022/03/10 16:01:01 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-10T16:01:01Z"} 2022/03/10 16:01:01 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-631431042/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T16:01:01Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-10T16:01:01Z"} {"filepath":"/tmp/helm-tmpl-631431042/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T16:01:01Z"} 2022/03/10 16:01:01 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-10T16:01:01Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"zealous_mahavira","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-10T16:01:01Z"} 10.233.71.221 - - [10/Mar/2022:16:01:01 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 305 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"zealous_mahavira","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-10T16:01:01Z"} 10.233.71.221 - - [10/Mar/2022:16:01:01 +0000] "GET /basicnf-owner/k8sregion/infra_workload/zealous_mahavira HTTP/1.1" 200 124 10.233.71.221 - - [10/Mar/2022:16:01:02 +0000] "POST /basicnf-owner/k8sregion/infra_workload/zealous_mahavira HTTP/1.1" 405 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"zealous_mahavira","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-10T16:01:41Z"} 10.233.71.221 - - [10/Mar/2022:16:01:41 +0000] "GET /basicnf-owner/k8sregion/infra_workload/zealous_mahavira HTTP/1.1" 200 124 2022/03/10 16:01:42 [Config Cleanup] Instance zealous_mahavira 2022/03/10 16:01:42 [scheduleResources]: START thread 2022/03/10 16:01:47 Stop config thread for zealous_mahavira 2022/03/10 16:01:47 [scheduleResources]: STOP thread {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-10T16:01:47Z"} 2022/03/10 16:01:47 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-10T16:01:47Z"} 2022/03/10 16:01:47 Deleting service: cnftest-nginx 2022/03/10 16:01:47 Deleted Obj with ID ObjectID("622a20bd1f5feb9b4b5dd6e2") 2022/03/10 16:01:47 Deleted Obj with ID ObjectID("622a20bd7bb8928c09aacd48") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"zealous_mahavira","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-03-10T16:01:47Z"} 10.233.71.221 - - [10/Mar/2022:16:01:42 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/zealous_mahavira HTTP/1.1" 202 124 10.233.71.221 - - [10/Mar/2022:16:02:34 +0000] "GET /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80 HTTP/1.1" 200 270 10.233.71.221 - - [10/Mar/2022:16:02:34 +0000] "GET /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/profile/cnftest HTTP/1.1" 200 240 2022/03/10 16:02:34 Deleted Obj with ID ObjectID("622a20a71f5feb9b4b5dd6e0") 2022/03/10 16:02:34 Deleted Obj with ID ObjectID("622a20a81f5feb9b4b5dd6e1") 10.233.71.221 - - [10/Mar/2022:16:02:34 +0000] "DELETE /v1/rb/definition/16d4c05b-5b6e-41bd-9cad-cc7f2a0fa98c/37b9be22-cb7b-40e0-93ac-4626529e4a80/profile/cnftest HTTP/1.1" 204 0 2022/03/10 16:02:34 Deleted Obj with ID ObjectID("622a20a87bb8928c09aacd1f") 10.233.71.221 - - [10/Mar/2022:16:03:30 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 2022/03/10 16:03:30 Deleted Obj with ID ObjectID("622a1e821f5feb9b4b5dd6c8") 2022/03/10 16:03:30 Deleted Obj with ID ObjectID("622a1e827bb8928c09aacb41") 10.233.71.221 - - [10/Mar/2022:16:03:30 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 10.233.71.221 - - [10/Mar/2022:16:06:03 +0000] "DELETE /v1/instance/vibrant_blackburn HTTP/1.1" 500 177 {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-10T16:06:03Z"} 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc/v1 HTTP/1.1" 200 125 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc/v1/content HTTP/1.1" 200 0 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc/v1 HTTP/1.1" 200 138 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc/v1/profile HTTP/1.1" 201 308 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc/v1/profile/pnf-cnf-profile-4f9b91e5-f725-4fdd-946f-4c10e4a688f7 HTTP/1.1" 200 308 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc/v1/profile/pnf-cnf-profile-4f9b91e5-f725-4fdd-946f-4c10e4a688f7/content HTTP/1.1" 200 0 2022/03/10 16:09:58 Main rss info 2022/03/10 16:09:58 Path: /tmp/helm-tmpl-607936421/manifest-0 2022/03/10 16:09:58 Kind: Service 2022/03/10 16:09:58 Path: /tmp/helm-tmpl-607936421/manifest-1 2022/03/10 16:09:58 Kind: Service 2022/03/10 16:09:58 Path: /tmp/helm-tmpl-607936421/manifest-2 2022/03/10 16:09:58 Kind: Pod 2022/03/10 16:09:58 Path: /tmp/helm-tmpl-607936421/manifest-3 2022/03/10 16:09:58 Kind: Deployment 2022/03/10 16:09:58 Crd rss info 2022/03/10 16:09:58 Hook info {"filepath":"/tmp/helm-tmpl-607936421/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-10T16:09:58Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-607936421/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-10T16:09:58Z"} 2022/03/10 16:09:58 Instance: adoring_ride, Main rss are failed, skip post-install and remove instance in DB 2022/03/10 16:09:58 Deleted Obj with ID ObjectID("622a22d61f5feb9b4b5dd6eb") 2022/03/10 16:09:58 Deleted Obj with ID ObjectID("622a22d67bb8928c09aacefa") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-a8e4829d-a242-4323-b4ce-807f837b74dc","rb-version":"v1","profile-name":"pnf-cnf-profile-4f9b91e5-f725-4fdd-946f-4c10e4a688f7","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-10T16:09:58Z"} 10.233.71.221 - - [10/Mar/2022:16:09:58 +0000] "POST /v1/instance HTTP/1.1" 500 180