2021/10/15 05:44:00 Starting Kubernetes Multicloud API 2021/10/15 05:44:00 Error Getting TLS Configuration. Starting without TLS... 10.233.70.214 - - [15/Oct/2021:08:05:05 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [15/Oct/2021:08:07:51 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [15/Oct/2021:08:14:05 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [15/Oct/2021:08:31:42 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 127.0.0.1 - - [15/Oct/2021:08:32:03 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.70.214 - - [15/Oct/2021:08:32:12 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.70.214 - - [15/Oct/2021:08:32:12 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7561 10.233.70.214 - - [15/Oct/2021:08:32:13 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7561 127.0.0.1 - - [15/Oct/2021:08:32:26 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [15/Oct/2021:08:32:26 +0000] "POST /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/content HTTP/1.1" 200 0 10.233.70.214 - - [15/Oct/2021:08:39:02 +0000] "GET /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6 HTTP/1.1" 200 270 10.233.70.214 - - [15/Oct/2021:08:39:02 +0000] "GET /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/profile/cnftest HTTP/1.1" 404 87 10.233.70.214 - - [15/Oct/2021:08:39:02 +0000] "POST /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/profile HTTP/1.1" 201 212 10.233.70.214 - - [15/Oct/2021:08:39:02 +0000] "GET /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/profile/cnftest HTTP/1.1" 200 212 10.233.70.214 - - [15/Oct/2021:08:39:02 +0000] "POST /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/profile/cnftest/content HTTP/1.1" 200 0 10.233.70.214 - - [15/Oct/2021:08:39:16 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-94840713-fb30-49cc-836a-e8866b0367d4 HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2021-10-15T08:39:16Z"} {"Body":{},"Payload":{"generic-vnf-id":"56fdf2ff-8978-4336-87cf-aa64b35dc4af","vf-module-id":"80c90d2d-a7f9-4e68-a221-aad8c9f54da9","vf-module-model-invariant-id":"90871bff-177d-4e83-820f-7cf751261d24","vf-module-model-version-id":"d2d1e511-19a8-4e08-a427-82df793a39a3","vf-module-model-customization-id":"e67fe002-2e25-4a39-b816-a5fe1955cbe6","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-94840713-fb30-49cc-836a-e8866b0367d4"}},"level":"info","msg":"Broker API Payload","time":"2021-10-15T08:39:16Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"90871bff-177d-4e83-820f-7cf751261d24","rb-version":"e67fe002-2e25-4a39-b816-a5fe1955cbe6","profile-name":"cnftest","release-name":"80c90d2d-a7f9-4e68-a221-aad8c9f54da9","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-94840713-fb30-49cc-836a-e8866b0367d4"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2021-10-15T08:39:16Z"} 2021/10/15 08:39:16 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2021/10/15 08:39:16 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2021/10/15 08:39:18 Main rss info 2021/10/15 08:39:18 Path: /tmp/helm-tmpl-838307372/manifest-0 2021/10/15 08:39:18 Kind: Service 2021/10/15 08:39:18 Path: /tmp/helm-tmpl-838307372/manifest-1 2021/10/15 08:39:18 Kind: Deployment 2021/10/15 08:39:18 Crd rss info 2021/10/15 08:39:18 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2021-10-15T08:39:18Z"} 2021/10/15 08:39:18 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-838307372/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:39:18Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2021-10-15T08:39:19Z"} {"filepath":"/tmp/helm-tmpl-838307372/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:39:19Z"} 2021/10/15 08:39:19 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2021-10-15T08:39:19Z"} 10.233.70.214 - - [15/Oct/2021:08:39:16 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 310 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"compassionate_elgamal","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2021-10-15T08:39:19Z"} 10.233.70.214 - - [15/Oct/2021:08:39:19 +0000] "GET /basicnf-owner/k8sregion/infra_workload/compassionate_elgamal HTTP/1.1" 200 129 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"compassionate_elgamal","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2021-10-15T08:39:19Z"} 10.233.70.214 - - [15/Oct/2021:08:39:19 +0000] "POST /basicnf-owner/k8sregion/infra_workload/compassionate_elgamal HTTP/1.1" 405 0 10.233.70.214 - - [15/Oct/2021:08:39:48 +0000] "GET /basicnf-owner/k8sregion/infra_workload/compassionate_elgamal HTTP/1.1" 200 129 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"compassionate_elgamal","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2021-10-15T08:39:48Z"} 2021/10/15 08:39:48 [Config Cleanup] Instance compassionate_elgamal 2021/10/15 08:39:48 [scheduleResources]: START thread 2021/10/15 08:39:53 Stop config thread for compassionate_elgamal {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2021-10-15T08:39:53Z"} 2021/10/15 08:39:53 No plugin for kind Deployment found. Using generic Plugin 2021/10/15 08:39:53 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2021-10-15T08:39:53Z"} 2021/10/15 08:39:53 Deleting service: cnftest-nginx 2021/10/15 08:39:54 Deleted Obj with ID ObjectID("61693e360b5c7fa14802b116") 2021/10/15 08:39:54 Deleted Obj with ID ObjectID("61693e36e70cc7f1d4bae915") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"compassionate_elgamal","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2021-10-15T08:39:54Z"} 10.233.70.214 - - [15/Oct/2021:08:39:48 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/compassionate_elgamal HTTP/1.1" 202 129 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7561 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "GET /v1/rb/definition/pnf-cnf-rb-0b50b23c-5fc0-4789-995c-c1a58ff078ec/v1 HTTP/1.1" 200 125 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "POST /v1/rb/definition/pnf-cnf-rb-0b50b23c-5fc0-4789-995c-c1a58ff078ec/v1/content HTTP/1.1" 200 0 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "GET /v1/rb/definition/pnf-cnf-rb-0b50b23c-5fc0-4789-995c-c1a58ff078ec/v1 HTTP/1.1" 200 138 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "POST /v1/rb/definition/pnf-cnf-rb-0b50b23c-5fc0-4789-995c-c1a58ff078ec/v1/profile HTTP/1.1" 201 280 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "GET /v1/rb/definition/pnf-cnf-rb-0b50b23c-5fc0-4789-995c-c1a58ff078ec/v1/profile/pnf-cnf-profile-2c7039bc-53ee-4e6e-b986-79918176bbff HTTP/1.1" 200 280 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "POST /v1/rb/definition/pnf-cnf-rb-0b50b23c-5fc0-4789-995c-c1a58ff078ec/v1/profile/pnf-cnf-profile-2c7039bc-53ee-4e6e-b986-79918176bbff/content HTTP/1.1" 200 0 2021/10/15 08:40:00 Main rss info 2021/10/15 08:40:00 Path: /tmp/helm-tmpl-758201299/manifest-0 2021/10/15 08:40:00 Kind: Service 2021/10/15 08:40:00 Path: /tmp/helm-tmpl-758201299/manifest-1 2021/10/15 08:40:00 Kind: Service 2021/10/15 08:40:00 Path: /tmp/helm-tmpl-758201299/manifest-2 2021/10/15 08:40:00 Kind: Pod 2021/10/15 08:40:00 Path: /tmp/helm-tmpl-758201299/manifest-3 2021/10/15 08:40:00 Kind: Deployment 2021/10/15 08:40:00 Crd rss info 2021/10/15 08:40:00 Hook info {"filepath":"/tmp/helm-tmpl-758201299/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:40:02Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2021-10-15T08:40:02Z"} {"filepath":"/tmp/helm-tmpl-758201299/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:40:02Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2021-10-15T08:40:03Z"} {"filepath":"/tmp/helm-tmpl-758201299/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:40:03Z"} 2021/10/15 08:40:03 No plugin for kind Pod found. Using generic Plugin 2021/10/15 08:40:03 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2021-10-15T08:40:03Z"} {"filepath":"/tmp/helm-tmpl-758201299/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:40:03Z"} 2021/10/15 08:40:03 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2021-10-15T08:40:04Z"} 10.233.70.214 - - [15/Oct/2021:08:40:00 +0000] "POST /v1/instance HTTP/1.1" 201 733 10.233.70.214 - - [15/Oct/2021:08:40:04 +0000] "GET /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6 HTTP/1.1" 200 270 10.233.70.214 - - [15/Oct/2021:08:40:04 +0000] "GET /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/profile/cnftest HTTP/1.1" 200 212 2021/10/15 08:40:04 Deleted Obj with ID ObjectID("61693e260b5c7fa14802b113") 2021/10/15 08:40:04 Deleted Obj with ID ObjectID("61693e260b5c7fa14802b114") 2021/10/15 08:40:04 Deleted Obj with ID ObjectID("61693e26e70cc7f1d4bae8ea") 10.233.70.214 - - [15/Oct/2021:08:40:04 +0000] "DELETE /v1/rb/definition/90871bff-177d-4e83-820f-7cf751261d24/e67fe002-2e25-4a39-b816-a5fe1955cbe6/profile/cnftest HTTP/1.1" 204 0 10.233.70.214 - - [15/Oct/2021:08:40:34 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7561 2021/10/15 08:40:34 Deleted Obj with ID ObjectID("61693c8c0b5c7fa14802b10d") 2021/10/15 08:40:34 Deleted Obj with ID ObjectID("61693c8de70cc7f1d4bae7d0") 10.233.70.214 - - [15/Oct/2021:08:40:34 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 127.0.0.1 - - [15/Oct/2021:08:41:23 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.70.214 - - [15/Oct/2021:08:47:25 +0000] "DELETE /v1/instance/affectionate_spence HTTP/1.1" 500 177 {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2021-10-15T08:47:25Z"} 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7561 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7561 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "GET /v1/rb/definition/pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b/v1 HTTP/1.1" 200 125 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "POST /v1/rb/definition/pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b/v1/content HTTP/1.1" 200 0 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "GET /v1/rb/definition/pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b/v1 HTTP/1.1" 200 138 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "POST /v1/rb/definition/pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b/v1/profile HTTP/1.1" 201 280 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "GET /v1/rb/definition/pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b/v1/profile/pnf-cnf-profile-a66a7be7-5120-458a-81b5-f2759651a2f9 HTTP/1.1" 200 280 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "POST /v1/rb/definition/pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b/v1/profile/pnf-cnf-profile-a66a7be7-5120-458a-81b5-f2759651a2f9/content HTTP/1.1" 200 0 2021/10/15 08:53:45 Main rss info 2021/10/15 08:53:45 Path: /tmp/helm-tmpl-283161258/manifest-0 2021/10/15 08:53:45 Kind: Service 2021/10/15 08:53:45 Path: /tmp/helm-tmpl-283161258/manifest-1 2021/10/15 08:53:45 Kind: Service 2021/10/15 08:53:45 Path: /tmp/helm-tmpl-283161258/manifest-2 2021/10/15 08:53:45 Kind: Pod 2021/10/15 08:53:45 Path: /tmp/helm-tmpl-283161258/manifest-3 2021/10/15 08:53:45 Kind: Deployment 2021/10/15 08:53:45 Crd rss info 2021/10/15 08:53:45 Hook info {"filepath":"/tmp/helm-tmpl-283161258/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-15T08:53:45Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-283161258/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2021-10-15T08:53:45Z"} 2021/10/15 08:53:45 Instance: loving_elgamal, Main rss are failed, skip post-install and remove instance in DB 2021/10/15 08:53:45 Deleted Obj with ID ObjectID("616941990b5c7fa14802b12a") 2021/10/15 08:53:45 Deleted Obj with ID ObjectID("61694199e70cc7f1d4baebd6") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-aa68f9e6-55fc-4092-9b4a-83685759ea9b","rb-version":"v1","profile-name":"pnf-cnf-profile-a66a7be7-5120-458a-81b5-f2759651a2f9","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2021-10-15T08:53:45Z"} 10.233.70.214 - - [15/Oct/2021:08:53:44 +0000] "POST /v1/instance HTTP/1.1" 500 180