2022/03/17 09:26:08 Starting Kubernetes Multicloud API 2022/03/17 09:26:08 Error Getting TLS Configuration. Starting without TLS... 10.233.68.196 - - [17/Mar/2022:10:39:21 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [17/Mar/2022:10:42:07 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [17/Mar/2022:10:43:47 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [17/Mar/2022:10:46:47 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.68.196 - - [17/Mar/2022:10:54:54 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.68.196 - - [17/Mar/2022:10:54:54 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7665 10.233.68.196 - - [17/Mar/2022:10:54:54 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 10.233.68.196 - - [17/Mar/2022:10:54:54 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "GET /v1/rb/definition/pnf-cnf-rb-1915ec78-802c-446c-85a2-a7e2a1861ac2/v1 HTTP/1.1" 200 125 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "POST /v1/rb/definition/pnf-cnf-rb-1915ec78-802c-446c-85a2-a7e2a1861ac2/v1/content HTTP/1.1" 200 0 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "GET /v1/rb/definition/pnf-cnf-rb-1915ec78-802c-446c-85a2-a7e2a1861ac2/v1 HTTP/1.1" 200 138 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "POST /v1/rb/definition/pnf-cnf-rb-1915ec78-802c-446c-85a2-a7e2a1861ac2/v1/profile HTTP/1.1" 201 308 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "GET /v1/rb/definition/pnf-cnf-rb-1915ec78-802c-446c-85a2-a7e2a1861ac2/v1/profile/pnf-cnf-profile-97756479-c213-4920-b047-84a122802044 HTTP/1.1" 200 308 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "POST /v1/rb/definition/pnf-cnf-rb-1915ec78-802c-446c-85a2-a7e2a1861ac2/v1/profile/pnf-cnf-profile-97756479-c213-4920-b047-84a122802044/content HTTP/1.1" 200 0 2022/03/17 10:54:55 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/17 10:54:55 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/17 10:54:55 Main rss info 2022/03/17 10:54:55 Path: /tmp/helm-tmpl-347111091/manifest-0 2022/03/17 10:54:55 Kind: Service 2022/03/17 10:54:55 Path: /tmp/helm-tmpl-347111091/manifest-1 2022/03/17 10:54:55 Kind: Service 2022/03/17 10:54:55 Path: /tmp/helm-tmpl-347111091/manifest-2 2022/03/17 10:54:55 Kind: Pod 2022/03/17 10:54:55 Path: /tmp/helm-tmpl-347111091/manifest-3 2022/03/17 10:54:55 Kind: Deployment 2022/03/17 10:54:55 Crd rss info 2022/03/17 10:54:55 Hook info {"filepath":"/tmp/helm-tmpl-347111091/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T10:54:55Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-17T10:54:55Z"} {"filepath":"/tmp/helm-tmpl-347111091/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T10:54:55Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-17T10:54:55Z"} {"filepath":"/tmp/helm-tmpl-347111091/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T10:54:55Z"} 2022/03/17 10:54:55 No plugin for kind Pod found. Using generic Plugin 2022/03/17 10:54:55 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-17T10:54:55Z"} {"filepath":"/tmp/helm-tmpl-347111091/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T10:54:55Z"} 2022/03/17 10:54:55 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-17T10:54:55Z"} 10.233.68.196 - - [17/Mar/2022:10:54:55 +0000] "POST /v1/instance HTTP/1.1" 201 728 10.233.68.196 - - [17/Mar/2022:10:57:16 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 127.0.0.1 - - [17/Mar/2022:10:57:31 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [17/Mar/2022:10:57:31 +0000] "POST /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/content HTTP/1.1" 200 0 127.0.0.1 - - [17/Mar/2022:10:57:44 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 127.0.0.1 - - [17/Mar/2022:10:58:28 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.68.196 - - [17/Mar/2022:11:02:09 +0000] "GET /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e HTTP/1.1" 200 270 10.233.68.196 - - [17/Mar/2022:11:02:09 +0000] "GET /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/profile/cnftest HTTP/1.1" 404 87 10.233.68.196 - - [17/Mar/2022:11:02:09 +0000] "POST /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/profile HTTP/1.1" 201 240 10.233.68.196 - - [17/Mar/2022:11:02:09 +0000] "GET /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/profile/cnftest HTTP/1.1" 200 240 10.233.68.196 - - [17/Mar/2022:11:02:09 +0000] "POST /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/profile/cnftest/content HTTP/1.1" 200 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-17T11:02:28Z"} 10.233.68.196 - - [17/Mar/2022:11:02:28 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-1db41ba3-cabd-47a9-9361-22e9ad9a2a82 HTTP/1.1" 200 114 {"Body":{},"Payload":{"generic-vnf-id":"79af0332-1ff7-4dd6-bf57-c66d54a68429","vf-module-id":"7c9b5342-26c4-4fa7-89e5-da3117515a50","vf-module-model-invariant-id":"0cd458a6-60a8-4158-af52-5b85f5eb978d","vf-module-model-version-id":"f4843919-7e08-4485-8055-31dea83c59bb","vf-module-model-customization-id":"27400d5b-6ba2-4a97-b151-2dd4231c081e","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-1db41ba3-cabd-47a9-9361-22e9ad9a2a82"}},"level":"info","msg":"Broker API Payload","time":"2022-03-17T11:02:28Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"0cd458a6-60a8-4158-af52-5b85f5eb978d","rb-version":"27400d5b-6ba2-4a97-b151-2dd4231c081e","profile-name":"cnftest","release-name":"7c9b5342-26c4-4fa7-89e5-da3117515a50","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-1db41ba3-cabd-47a9-9361-22e9ad9a2a82"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-17T11:02:28Z"} 2022/03/17 11:02:28 Main rss info 2022/03/17 11:02:28 Path: /tmp/helm-tmpl-598322239/manifest-0 2022/03/17 11:02:28 Kind: Service 2022/03/17 11:02:28 Path: /tmp/helm-tmpl-598322239/manifest-1 2022/03/17 11:02:28 Kind: Deployment 2022/03/17 11:02:28 Crd rss info 2022/03/17 11:02:28 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-17T11:02:28Z"} 2022/03/17 11:02:28 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-598322239/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T11:02:28Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-17T11:02:28Z"} {"filepath":"/tmp/helm-tmpl-598322239/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T11:02:28Z"} 2022/03/17 11:02:28 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-17T11:02:28Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_cerf","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-17T11:02:28Z"} 10.233.68.196 - - [17/Mar/2022:11:02:28 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 299 10.233.68.196 - - [17/Mar/2022:11:02:29 +0000] "GET /basicnf-owner/k8sregion/infra_workload/brave_cerf HTTP/1.1" 200 118 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_cerf","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-17T11:02:29Z"} 10.233.68.196 - - [17/Mar/2022:11:02:29 +0000] "POST /basicnf-owner/k8sregion/infra_workload/brave_cerf HTTP/1.1" 405 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_cerf","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-17T11:03:06Z"} 10.233.68.196 - - [17/Mar/2022:11:03:06 +0000] "GET /basicnf-owner/k8sregion/infra_workload/brave_cerf HTTP/1.1" 200 118 2022/03/17 11:03:06 [Config Cleanup] Instance brave_cerf 2022/03/17 11:03:06 [scheduleResources]: START thread 2022/03/17 11:03:11 Stop config thread for brave_cerf {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-17T11:03:11Z"} 2022/03/17 11:03:11 No plugin for kind Deployment found. Using generic Plugin 2022/03/17 11:03:11 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-17T11:03:11Z"} 2022/03/17 11:03:11 Deleting service: cnftest-nginx 2022/03/17 11:03:11 Deleted Obj with ID ObjectID("62331544818a012e50903e21") 2022/03/17 11:03:11 Deleted Obj with ID ObjectID("623315447007e1a3a362b077") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_cerf","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-03-17T11:03:11Z"} 10.233.68.196 - - [17/Mar/2022:11:03:06 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/brave_cerf HTTP/1.1" 202 118 10.233.68.196 - - [17/Mar/2022:11:03:54 +0000] "GET /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e HTTP/1.1" 200 270 10.233.68.196 - - [17/Mar/2022:11:03:54 +0000] "GET /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/profile/cnftest HTTP/1.1" 200 240 2022/03/17 11:03:54 Deleted Obj with ID ObjectID("62331531818a012e50903e1f") 2022/03/17 11:03:54 Deleted Obj with ID ObjectID("62331531818a012e50903e20") 2022/03/17 11:03:54 Deleted Obj with ID ObjectID("623315317007e1a3a362b04e") 10.233.68.196 - - [17/Mar/2022:11:03:54 +0000] "DELETE /v1/rb/definition/0cd458a6-60a8-4158-af52-5b85f5eb978d/27400d5b-6ba2-4a97-b151-2dd4231c081e/profile/cnftest HTTP/1.1" 204 0 10.233.68.196 - - [17/Mar/2022:11:04:27 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 2022/03/17 11:04:27 Deleted Obj with ID ObjectID("6233137e818a012e50903e0a") 10.233.68.196 - - [17/Mar/2022:11:04:27 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/17 11:04:27 Deleted Obj with ID ObjectID("6233137e7007e1a3a362aec1") {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-17T11:06:49Z"} 10.233.68.196 - - [17/Mar/2022:11:06:49 +0000] "DELETE /v1/instance/dreamy_kapitsa HTTP/1.1" 500 177 10.233.68.196 - - [17/Mar/2022:11:10:25 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7665 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7665 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "GET /v1/rb/definition/pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c/v1 HTTP/1.1" 200 125 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "POST /v1/rb/definition/pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c/v1/content HTTP/1.1" 200 0 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "GET /v1/rb/definition/pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c/v1 HTTP/1.1" 200 138 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "POST /v1/rb/definition/pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c/v1/profile HTTP/1.1" 201 308 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "GET /v1/rb/definition/pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c/v1/profile/pnf-cnf-profile-40737de0-cbb7-490f-a1fd-dfce11500b38 HTTP/1.1" 200 308 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "POST /v1/rb/definition/pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c/v1/profile/pnf-cnf-profile-40737de0-cbb7-490f-a1fd-dfce11500b38/content HTTP/1.1" 200 0 2022/03/17 11:10:26 Main rss info 2022/03/17 11:10:26 Path: /tmp/helm-tmpl-556294669/manifest-0 2022/03/17 11:10:26 Kind: Service 2022/03/17 11:10:26 Path: /tmp/helm-tmpl-556294669/manifest-1 2022/03/17 11:10:26 Kind: Service 2022/03/17 11:10:26 Path: /tmp/helm-tmpl-556294669/manifest-2 2022/03/17 11:10:26 Kind: Pod 2022/03/17 11:10:26 Path: /tmp/helm-tmpl-556294669/manifest-3 2022/03/17 11:10:26 Kind: Deployment 2022/03/17 11:10:26 Crd rss info 2022/03/17 11:10:26 Hook info {"filepath":"/tmp/helm-tmpl-556294669/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-17T11:10:26Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-556294669/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-17T11:10:26Z"} 2022/03/17 11:10:26 Instance: eager_galileo, Main rss are failed, skip post-install and remove instance in DB 2022/03/17 11:10:26 Deleted Obj with ID ObjectID("62331722818a012e50903e2a") 2022/03/17 11:10:26 Deleted Obj with ID ObjectID("623317227007e1a3a362b1fd") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-bf8ae3e7-67a5-4a7b-8854-2cf4eb0f3d6c","rb-version":"v1","profile-name":"pnf-cnf-profile-40737de0-cbb7-490f-a1fd-dfce11500b38","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-17T11:10:26Z"} 10.233.68.196 - - [17/Mar/2022:11:10:26 +0000] "POST /v1/instance HTTP/1.1" 500 180