2022/01/04 02:11:38 Starting Kubernetes Multicloud API 2022/01/04 02:11:38 Error Getting TLS Configuration. Starting without TLS... 127.0.0.1 - - [04/Jan/2022:05:13:03 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 10.233.77.75 - - [04/Jan/2022:05:17:02 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:18:22 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [04/Jan/2022:05:18:27 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.77.75 - - [04/Jan/2022:05:19:54 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:20:22 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:21:32 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:22:49 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [04/Jan/2022:05:24:43 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.77.75 - - [04/Jan/2022:05:25:36 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:34:57 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7565 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-5044aab2-4790-4138-a654-15e3cd98c94b/v1 HTTP/1.1" 200 125 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-5044aab2-4790-4138-a654-15e3cd98c94b/v1/content HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-5044aab2-4790-4138-a654-15e3cd98c94b/v1 HTTP/1.1" 200 138 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-5044aab2-4790-4138-a654-15e3cd98c94b/v1/profile HTTP/1.1" 201 314 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-5044aab2-4790-4138-a654-15e3cd98c94b/v1/profile/pnf-cnf-profile-6fa35705-9c0b-45f7-843a-6109dbf81f84 HTTP/1.1" 200 314 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-5044aab2-4790-4138-a654-15e3cd98c94b/v1/profile/pnf-cnf-profile-6fa35705-9c0b-45f7-843a-6109dbf81f84/content HTTP/1.1" 200 0 2022/01/04 05:34:58 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/01/04 05:34:58 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/01/04 05:34:59 Main rss info 2022/01/04 05:34:59 Path: /tmp/helm-tmpl-085545147/manifest-0 2022/01/04 05:34:59 Kind: Service 2022/01/04 05:34:59 Path: /tmp/helm-tmpl-085545147/manifest-1 2022/01/04 05:34:59 Kind: Service 2022/01/04 05:34:59 Path: /tmp/helm-tmpl-085545147/manifest-2 2022/01/04 05:34:59 Kind: Pod 2022/01/04 05:34:59 Path: /tmp/helm-tmpl-085545147/manifest-3 2022/01/04 05:34:59 Kind: Deployment 2022/01/04 05:34:59 Crd rss info 2022/01/04 05:34:59 Hook info {"level":"info","msg":"Creating Namespace","namespace":"onap-tests","time":"2022-01-04T05:34:59Z"} 2022/01/04 05:34:59 Namespace (onap-tests) created {"filepath":"/tmp/helm-tmpl-085545147/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:34:59Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-01-04T05:34:59Z"} {"filepath":"/tmp/helm-tmpl-085545147/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:34:59Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-01-04T05:34:59Z"} {"filepath":"/tmp/helm-tmpl-085545147/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:34:59Z"} 2022/01/04 05:34:59 No plugin for kind Pod found. Using generic Plugin 2022/01/04 05:34:59 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-01-04T05:34:59Z"} {"filepath":"/tmp/helm-tmpl-085545147/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:34:59Z"} 2022/01/04 05:34:59 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-01-04T05:34:59Z"} 10.233.77.75 - - [04/Jan/2022:05:34:58 +0000] "POST /v1/instance HTTP/1.1" 201 731 127.0.0.1 - - [04/Jan/2022:05:36:04 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [04/Jan/2022:05:36:23 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.77.75 - - [04/Jan/2022:05:40:04 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 127.0.0.1 - - [04/Jan/2022:05:40:14 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [04/Jan/2022:05:40:14 +0000] "POST /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/content HTTP/1.1" 200 0 127.0.0.1 - - [04/Jan/2022:05:40:27 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 10.233.77.75 - - [04/Jan/2022:05:43:35 +0000] "GET /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047 HTTP/1.1" 200 270 10.233.77.75 - - [04/Jan/2022:05:43:35 +0000] "GET /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/profile/cnftest HTTP/1.1" 404 87 10.233.77.75 - - [04/Jan/2022:05:43:35 +0000] "POST /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/profile HTTP/1.1" 201 240 10.233.77.75 - - [04/Jan/2022:05:43:35 +0000] "GET /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/profile/cnftest HTTP/1.1" 200 240 10.233.77.75 - - [04/Jan/2022:05:43:35 +0000] "POST /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/profile/cnftest/content HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:43:43 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-b4c7201e-ef68-46db-9d22-42e90066c186 HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-01-04T05:43:43Z"} {"Body":{},"Payload":{"generic-vnf-id":"6e819655-9cb4-4663-94f5-fb1a4db258c7","vf-module-id":"a1733252-4578-457a-8e91-38eb1e0c389a","vf-module-model-invariant-id":"4c9b59ba-9015-4f55-a168-c362022216eb","vf-module-model-version-id":"1dd36668-c33f-4787-b0df-cc4eabeb16dc","vf-module-model-customization-id":"bcb59562-d598-4a60-8ec1-4b6b17f4b047","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-b4c7201e-ef68-46db-9d22-42e90066c186"}},"level":"info","msg":"Broker API Payload","time":"2022-01-04T05:43:43Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"4c9b59ba-9015-4f55-a168-c362022216eb","rb-version":"bcb59562-d598-4a60-8ec1-4b6b17f4b047","profile-name":"cnftest","release-name":"a1733252-4578-457a-8e91-38eb1e0c389a","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-b4c7201e-ef68-46db-9d22-42e90066c186"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-01-04T05:43:43Z"} 2022/01/04 05:43:43 Main rss info 2022/01/04 05:43:43 Path: /tmp/helm-tmpl-783777870/manifest-0 2022/01/04 05:43:43 Kind: Service 2022/01/04 05:43:43 Path: /tmp/helm-tmpl-783777870/manifest-1 2022/01/04 05:43:43 Kind: Deployment 2022/01/04 05:43:43 Crd rss info 2022/01/04 05:43:43 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-01-04T05:43:43Z"} 2022/01/04 05:43:43 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-783777870/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:43:43Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-01-04T05:43:44Z"} {"filepath":"/tmp/helm-tmpl-783777870/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:43:44Z"} 2022/01/04 05:43:44 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-01-04T05:43:44Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"sleepy_kirch","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-01-04T05:43:44Z"} 10.233.77.75 - - [04/Jan/2022:05:43:43 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 301 10.233.77.75 - - [04/Jan/2022:05:43:44 +0000] "GET /basicnf-owner/k8sregion/infra_workload/sleepy_kirch HTTP/1.1" 200 120 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"sleepy_kirch","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-01-04T05:43:44Z"} 10.233.77.75 - - [04/Jan/2022:05:43:44 +0000] "POST /basicnf-owner/k8sregion/infra_workload/sleepy_kirch HTTP/1.1" 405 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"sleepy_kirch","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-01-04T05:44:07Z"} 10.233.77.75 - - [04/Jan/2022:05:44:07 +0000] "GET /basicnf-owner/k8sregion/infra_workload/sleepy_kirch HTTP/1.1" 200 120 2022/01/04 05:44:07 [Config Cleanup] Instance sleepy_kirch 2022/01/04 05:44:07 [scheduleResources]: START thread 2022/01/04 05:44:12 Stop config thread for sleepy_kirch {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-01-04T05:44:12Z"} 2022/01/04 05:44:12 [scheduleResources]: STOP thread 2022/01/04 05:44:12 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-01-04T05:44:12Z"} 2022/01/04 05:44:12 Deleting service: cnftest-nginx 2022/01/04 05:44:12 Deleted Obj with ID ObjectID("61d3de8f93151ac3bcc209ce") 2022/01/04 05:44:12 Deleted Obj with ID ObjectID("61d3de8fa1d4d81343a27a2c") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"sleepy_kirch","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-01-04T05:44:12Z"} 10.233.77.75 - - [04/Jan/2022:05:44:07 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/sleepy_kirch HTTP/1.1" 202 120 10.233.77.75 - - [04/Jan/2022:05:44:24 +0000] "GET /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047 HTTP/1.1" 200 270 10.233.77.75 - - [04/Jan/2022:05:44:24 +0000] "GET /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/profile/cnftest HTTP/1.1" 200 240 2022/01/04 05:44:24 Deleted Obj with ID ObjectID("61d3de8793151ac3bcc209cc") 2022/01/04 05:44:24 Deleted Obj with ID ObjectID("61d3de8793151ac3bcc209cd") 2022/01/04 05:44:24 Deleted Obj with ID ObjectID("61d3de87a1d4d81343a27a09") 10.233.77.75 - - [04/Jan/2022:05:44:24 +0000] "DELETE /v1/rb/definition/4c9b59ba-9015-4f55-a168-c362022216eb/bcb59562-d598-4a60-8ec1-4b6b17f4b047/profile/cnftest HTTP/1.1" 204 0 10.233.77.75 - - [04/Jan/2022:05:44:52 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 2022/01/04 05:44:52 Deleted Obj with ID ObjectID("61d3dc8293151ac3bcc209b4") 10.233.77.75 - - [04/Jan/2022:05:44:52 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/01/04 05:44:52 Deleted Obj with ID ObjectID("61d3dc82a1d4d81343a2783e") {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-01-04T05:46:16Z"} 10.233.77.75 - - [04/Jan/2022:05:46:16 +0000] "DELETE /v1/instance/cranky_nash HTTP/1.1" 500 177 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7565 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7565 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "GET /v1/rb/definition/pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc/v1 HTTP/1.1" 200 125 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "POST /v1/rb/definition/pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc/v1/content HTTP/1.1" 200 0 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "GET /v1/rb/definition/pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc/v1 HTTP/1.1" 200 138 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "POST /v1/rb/definition/pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc/v1/profile HTTP/1.1" 201 314 10.233.77.75 - - [04/Jan/2022:05:51:26 +0000] "GET /v1/rb/definition/pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc/v1/profile/pnf-cnf-profile-5711c401-7226-47ae-a0d6-11f9122810f3 HTTP/1.1" 200 314 10.233.77.75 - - [04/Jan/2022:05:51:27 +0000] "POST /v1/rb/definition/pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc/v1/profile/pnf-cnf-profile-5711c401-7226-47ae-a0d6-11f9122810f3/content HTTP/1.1" 200 0 2022/01/04 05:51:27 Main rss info 2022/01/04 05:51:27 Path: /tmp/helm-tmpl-842990820/manifest-0 2022/01/04 05:51:27 Kind: Service 2022/01/04 05:51:27 Path: /tmp/helm-tmpl-842990820/manifest-1 2022/01/04 05:51:27 Kind: Service 2022/01/04 05:51:27 Path: /tmp/helm-tmpl-842990820/manifest-2 2022/01/04 05:51:27 Kind: Pod 2022/01/04 05:51:27 Path: /tmp/helm-tmpl-842990820/manifest-3 2022/01/04 05:51:27 Kind: Deployment 2022/01/04 05:51:27 Crd rss info 2022/01/04 05:51:27 Hook info {"filepath":"/tmp/helm-tmpl-842990820/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-01-04T05:51:27Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-842990820/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-01-04T05:51:27Z"} 2022/01/04 05:51:27 Instance: gracious_rubin, Main rss are failed, skip post-install and remove instance in DB 2022/01/04 05:51:27 Deleted Obj with ID ObjectID("61d3e05f93151ac3bcc209d7") 2022/01/04 05:51:27 Deleted Obj with ID ObjectID("61d3e05fa1d4d81343a27bae") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-5e346a7c-58fe-4ae0-8b0c-991f99892fdc","rb-version":"v1","profile-name":"pnf-cnf-profile-5711c401-7226-47ae-a0d6-11f9122810f3","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-01-04T05:51:27Z"} 10.233.77.75 - - [04/Jan/2022:05:51:27 +0000] "POST /v1/instance HTTP/1.1" 500 180