2022/03/07 02:47:16 Starting Kubernetes Multicloud API 2022/03/07 02:47:16 Error Getting TLS Configuration. Starting without TLS... 10.233.65.193 - - [07/Mar/2022:03:58:14 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [07/Mar/2022:04:02:01 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [07/Mar/2022:04:02:16 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "GET /v1/rb/definition/pnf-cnf-rb-560381ab-e5dd-4b3a-a6d2-4c648a5f6d41/v1 HTTP/1.1" 200 125 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "POST /v1/rb/definition/pnf-cnf-rb-560381ab-e5dd-4b3a-a6d2-4c648a5f6d41/v1/content HTTP/1.1" 200 0 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "GET /v1/rb/definition/pnf-cnf-rb-560381ab-e5dd-4b3a-a6d2-4c648a5f6d41/v1 HTTP/1.1" 200 138 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "POST /v1/rb/definition/pnf-cnf-rb-560381ab-e5dd-4b3a-a6d2-4c648a5f6d41/v1/profile HTTP/1.1" 201 308 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "GET /v1/rb/definition/pnf-cnf-rb-560381ab-e5dd-4b3a-a6d2-4c648a5f6d41/v1/profile/pnf-cnf-profile-3ffe28db-ac7d-4d0a-b045-225034592771 HTTP/1.1" 200 308 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "POST /v1/rb/definition/pnf-cnf-rb-560381ab-e5dd-4b3a-a6d2-4c648a5f6d41/v1/profile/pnf-cnf-profile-3ffe28db-ac7d-4d0a-b045-225034592771/content HTTP/1.1" 200 0 2022/03/07 04:12:36 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/07 04:12:36 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/07 04:12:36 Main rss info 2022/03/07 04:12:36 Path: /tmp/helm-tmpl-346095767/manifest-0 2022/03/07 04:12:36 Kind: Service 2022/03/07 04:12:36 Path: /tmp/helm-tmpl-346095767/manifest-1 2022/03/07 04:12:36 Kind: Service 2022/03/07 04:12:36 Path: /tmp/helm-tmpl-346095767/manifest-2 2022/03/07 04:12:36 Kind: Pod 2022/03/07 04:12:36 Path: /tmp/helm-tmpl-346095767/manifest-3 2022/03/07 04:12:36 Kind: Deployment 2022/03/07 04:12:36 Crd rss info 2022/03/07 04:12:36 Hook info {"filepath":"/tmp/helm-tmpl-346095767/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:12:37Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-07T04:12:37Z"} {"filepath":"/tmp/helm-tmpl-346095767/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:12:37Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-07T04:12:37Z"} {"filepath":"/tmp/helm-tmpl-346095767/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:12:37Z"} 2022/03/07 04:12:37 No plugin for kind Pod found. Using generic Plugin 2022/03/07 04:12:37 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-07T04:12:37Z"} {"filepath":"/tmp/helm-tmpl-346095767/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:12:37Z"} 2022/03/07 04:12:37 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-07T04:12:37Z"} 10.233.65.193 - - [07/Mar/2022:04:12:36 +0000] "POST /v1/instance HTTP/1.1" 201 731 10.233.65.193 - - [07/Mar/2022:04:17:00 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 127.0.0.1 - - [07/Mar/2022:04:17:19 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [07/Mar/2022:04:17:19 +0000] "POST /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/content HTTP/1.1" 200 0 127.0.0.1 - - [07/Mar/2022:04:17:33 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 127.0.0.1 - - [07/Mar/2022:04:17:46 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [07/Mar/2022:04:18:10 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.65.193 - - [07/Mar/2022:04:21:52 +0000] "GET /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc HTTP/1.1" 200 270 10.233.65.193 - - [07/Mar/2022:04:21:52 +0000] "GET /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/profile/cnftest HTTP/1.1" 404 87 10.233.65.193 - - [07/Mar/2022:04:21:52 +0000] "POST /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/profile HTTP/1.1" 201 240 10.233.65.193 - - [07/Mar/2022:04:21:52 +0000] "GET /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/profile/cnftest HTTP/1.1" 200 240 10.233.65.193 - - [07/Mar/2022:04:21:52 +0000] "POST /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/profile/cnftest/content HTTP/1.1" 200 0 10.233.65.193 - - [07/Mar/2022:04:22:11 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-013a5446-d3b0-4d8b-aaaa-362ec79dca4f HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-07T04:22:11Z"} {"Body":{},"Payload":{"generic-vnf-id":"6d809b48-e34c-4cfb-950b-013aac5a32b8","vf-module-id":"35bf839b-aafd-455e-a49a-d4a0c55d31b3","vf-module-model-invariant-id":"ad01068f-42d0-48d6-85c2-b9bd32c2bd32","vf-module-model-version-id":"8f0a165c-8dc6-4243-bad7-58804799d9d3","vf-module-model-customization-id":"4f8a115e-67c9-4e38-9f93-2f5bab3d38dc","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-013a5446-d3b0-4d8b-aaaa-362ec79dca4f"}},"level":"info","msg":"Broker API Payload","time":"2022-03-07T04:22:12Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"ad01068f-42d0-48d6-85c2-b9bd32c2bd32","rb-version":"4f8a115e-67c9-4e38-9f93-2f5bab3d38dc","profile-name":"cnftest","release-name":"35bf839b-aafd-455e-a49a-d4a0c55d31b3","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-013a5446-d3b0-4d8b-aaaa-362ec79dca4f"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-07T04:22:12Z"} 2022/03/07 04:22:12 Main rss info 2022/03/07 04:22:12 Path: /tmp/helm-tmpl-150912227/manifest-0 2022/03/07 04:22:12 Kind: Service 2022/03/07 04:22:12 Path: /tmp/helm-tmpl-150912227/manifest-1 2022/03/07 04:22:12 Kind: Deployment 2022/03/07 04:22:12 Crd rss info 2022/03/07 04:22:12 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-07T04:22:12Z"} 2022/03/07 04:22:12 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-150912227/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:22:12Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-07T04:22:12Z"} {"filepath":"/tmp/helm-tmpl-150912227/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:22:12Z"} 2022/03/07 04:22:12 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-07T04:22:12Z"} 10.233.65.193 - - [07/Mar/2022:04:22:12 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 301 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"dreamy_hertz","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-07T04:22:12Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"dreamy_hertz","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-07T04:22:12Z"} 10.233.65.193 - - [07/Mar/2022:04:22:12 +0000] "GET /basicnf-owner/k8sregion/infra_workload/dreamy_hertz HTTP/1.1" 200 120 10.233.65.193 - - [07/Mar/2022:04:22:12 +0000] "POST /basicnf-owner/k8sregion/infra_workload/dreamy_hertz HTTP/1.1" 405 0 10.233.65.193 - - [07/Mar/2022:04:22:39 +0000] "GET /basicnf-owner/k8sregion/infra_workload/dreamy_hertz HTTP/1.1" 200 120 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"dreamy_hertz","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-07T04:22:39Z"} 2022/03/07 04:22:39 [Config Cleanup] Instance dreamy_hertz 2022/03/07 04:22:39 [scheduleResources]: START thread 2022/03/07 04:22:44 Stop config thread for dreamy_hertz {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-07T04:22:44Z"} 2022/03/07 04:22:44 No plugin for kind Deployment found. Using generic Plugin 2022/03/07 04:22:44 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-07T04:22:44Z"} 2022/03/07 04:22:44 Deleting service: cnftest-nginx 2022/03/07 04:22:45 Deleted Obj with ID ObjectID("622588746d98a43fef9ea236") 2022/03/07 04:22:45 Deleted Obj with ID ObjectID("62258874ece543a8fc9b2334") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"dreamy_hertz","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-03-07T04:22:45Z"} 10.233.65.193 - - [07/Mar/2022:04:22:39 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/dreamy_hertz HTTP/1.1" 202 120 10.233.65.193 - - [07/Mar/2022:04:23:03 +0000] "GET /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc HTTP/1.1" 200 270 10.233.65.193 - - [07/Mar/2022:04:23:03 +0000] "GET /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/profile/cnftest HTTP/1.1" 200 240 2022/03/07 04:23:03 Deleted Obj with ID ObjectID("622588606d98a43fef9ea234") 2022/03/07 04:23:03 Deleted Obj with ID ObjectID("622588606d98a43fef9ea235") 10.233.65.193 - - [07/Mar/2022:04:23:03 +0000] "DELETE /v1/rb/definition/ad01068f-42d0-48d6-85c2-b9bd32c2bd32/4f8a115e-67c9-4e38-9f93-2f5bab3d38dc/profile/cnftest HTTP/1.1" 204 0 2022/03/07 04:23:03 Deleted Obj with ID ObjectID("6225885fece543a8fc9b230b") 10.233.65.193 - - [07/Mar/2022:04:23:34 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 2022/03/07 04:23:34 Deleted Obj with ID ObjectID("622586346d98a43fef9ea21c") 10.233.65.193 - - [07/Mar/2022:04:23:34 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/07 04:23:34 Deleted Obj with ID ObjectID("62258634ece543a8fc9b212e") {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-07T04:25:55Z"} 10.233.65.193 - - [07/Mar/2022:04:25:55 +0000] "DELETE /v1/instance/dazzling_elbakyan HTTP/1.1" 500 177 10.233.65.193 - - [07/Mar/2022:04:28:42 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "GET /v1/rb/definition/pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59/v1 HTTP/1.1" 200 125 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "POST /v1/rb/definition/pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59/v1/content HTTP/1.1" 200 0 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "GET /v1/rb/definition/pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59/v1 HTTP/1.1" 200 138 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "POST /v1/rb/definition/pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59/v1/profile HTTP/1.1" 201 308 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "GET /v1/rb/definition/pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59/v1/profile/pnf-cnf-profile-18a16ebe-ccdb-4534-bc05-a6f76267b4b1 HTTP/1.1" 200 308 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "POST /v1/rb/definition/pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59/v1/profile/pnf-cnf-profile-18a16ebe-ccdb-4534-bc05-a6f76267b4b1/content HTTP/1.1" 200 0 2022/03/07 04:28:43 Main rss info 2022/03/07 04:28:43 Path: /tmp/helm-tmpl-740752386/manifest-0 2022/03/07 04:28:43 Kind: Service 2022/03/07 04:28:43 Path: /tmp/helm-tmpl-740752386/manifest-1 2022/03/07 04:28:43 Kind: Service 2022/03/07 04:28:43 Path: /tmp/helm-tmpl-740752386/manifest-2 2022/03/07 04:28:43 Kind: Pod 2022/03/07 04:28:43 Path: /tmp/helm-tmpl-740752386/manifest-3 2022/03/07 04:28:43 Kind: Deployment 2022/03/07 04:28:43 Crd rss info 2022/03/07 04:28:43 Hook info {"filepath":"/tmp/helm-tmpl-740752386/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-07T04:28:43Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-740752386/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-07T04:28:43Z"} 2022/03/07 04:28:43 Instance: epic_banach, Main rss are failed, skip post-install and remove instance in DB 2022/03/07 04:28:43 Deleted Obj with ID ObjectID("622589fb6d98a43fef9ea23f") 2022/03/07 04:28:43 Deleted Obj with ID ObjectID("622589fbece543a8fc9b2489") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-90c0e9f3-0d44-4dea-871d-21c904a28f59","rb-version":"v1","profile-name":"pnf-cnf-profile-18a16ebe-ccdb-4534-bc05-a6f76267b4b1","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-07T04:28:43Z"} 10.233.65.193 - - [07/Mar/2022:04:28:43 +0000] "POST /v1/instance HTTP/1.1" 500 180