By type
2022-04-15 01:46:31,044||140488452864800|ERROR|_utils|cotyledon._utils: [-] Unhandled exception
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/etcd/api.py", line 60, in get_client
return etcd3.client(host=self.host, port=self.port,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 1178, in client
return Etcd3Client(host=host,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 148, in __init__
resp = self.auth_stub.Authenticate(auth_request, self.timeout)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "etcdserver: request timed out"
debug_error_string = "{"created":"@1649987191.043244171","description":"Error received from peer ipv4:10.233.74.59:2379","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"etcdserver: request timed out","grpc_status":14}"
>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/cotyledon/_utils.py", line 95, in exit_on_exception
yield
File "/usr/local/lib/python3.9/site-packages/cotyledon/_service.py", line 139, in _run
self.run()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 477, in run
fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 440, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/futurist/_utils.py", line 49, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 334, in __check_for_messages
self._do()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 341, in _do
msgs = self.RPC.query.all()
File "/opt/has/conductor/conductor/common/music/model/search.py", line 80, in all
rows = db_backend.DB_API.row_read(**kwargs)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 136, in row_read
return self.get_values_prefix(key_prefix, pk_name, pk_value)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 82, in get_values_prefix
for kv in self.get_client().get_prefix(key_prefix)}
File "/opt/has/conductor/conductor/common/etcd/api.py", line 69, in get_client
raise EtcdClientException("Failed to establish connection with ETCD. GRPC {}".format(rpc_error.code()))
conductor.common.etcd.utils.EtcdClientException: Failed to establish connection with ETCD. GRPC StatusCode.UNAVAILABLE
2022-04-15 01:46:31,058||140488454986528|INFO|_service_manager|cotyledon._service_manager: [-] Child 209 exited with status 2
2022-04-15 02:20:20,451||140488452864800|ERROR|_utils|cotyledon._utils: [-] Unhandled exception
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/etcd/api.py", line 60, in get_client
return etcd3.client(host=self.host, port=self.port,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 1178, in client
return Etcd3Client(host=host,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 148, in __init__
resp = self.auth_stub.Authenticate(auth_request, self.timeout)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "etcdserver: request timed out"
debug_error_string = "{"created":"@1649989220.450168990","description":"Error received from peer ipv4:10.233.74.59:2379","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"etcdserver: request timed out","grpc_status":14}"
>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/cotyledon/_utils.py", line 95, in exit_on_exception
yield
File "/usr/local/lib/python3.9/site-packages/cotyledon/_service.py", line 139, in _run
self.run()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 477, in run
fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 440, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/futurist/_utils.py", line 49, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 334, in __check_for_messages
self._do()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 341, in _do
msgs = self.RPC.query.all()
File "/opt/has/conductor/conductor/common/music/model/search.py", line 80, in all
rows = db_backend.DB_API.row_read(**kwargs)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 136, in row_read
return self.get_values_prefix(key_prefix, pk_name, pk_value)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 82, in get_values_prefix
for kv in self.get_client().get_prefix(key_prefix)}
File "/opt/has/conductor/conductor/common/etcd/api.py", line 69, in get_client
raise EtcdClientException("Failed to establish connection with ETCD. GRPC {}".format(rpc_error.code()))
conductor.common.etcd.utils.EtcdClientException: Failed to establish connection with ETCD. GRPC StatusCode.UNAVAILABLE
2022-04-15 02:20:20,467||140488454986528|INFO|_service_manager|cotyledon._service_manager: [-] Child 1513 exited with status 2
2022-04-15 02:36:19,485||140488433507104|INFO|component|conductor.common.music.messaging.component: [-] Message 1e68dd11-d4a1-4496-8930-d65cb23b041f method plans_get received
2022-04-15 02:36:33,280||140488433507104|INFO|component|conductor.common.music.messaging.component: [-] Message 1e68dd11-d4a1-4496-8930-d65cb23b041f method plans_get, status: completed
2022-04-15 02:36:40,467||140488433507104|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:38:10,464||140488369593120|INFO|component|conductor.common.music.messaging.component: [-] Message 9f58f811-912e-4c9b-bbe9-48e6255c7f0e method plans_get received
2022-04-15 02:38:25,287||140488369593120|INFO|component|conductor.common.music.messaging.component: [-] Message 9f58f811-912e-4c9b-bbe9-48e6255c7f0e method plans_get, status: completed
2022-04-15 02:38:35,588||140488369593120|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:40:05,584||140488413350688|INFO|component|conductor.common.music.messaging.component: [-] Message 08c0d04f-d7c1-4835-9e45-b888340225f0 method plans_get received
2022-04-15 02:40:16,460||140488413350688|ERROR|component|conductor.common.music.messaging.component: [-] Exception during message handling
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/etcd/api.py", line 60, in get_client
return etcd3.client(host=self.host, port=self.port,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 1178, in client
return Etcd3Client(host=host,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 148, in __init__
resp = self.auth_stub.Authenticate(auth_request, self.timeout)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "etcdserver: request timed out"
debug_error_string = "{"created":"@1649990416.459616152","description":"Error received from peer ipv4:10.233.76.227:2379","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"etcdserver: request timed out","grpc_status":14}"
>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 413, in _do
result = method(msg.ctxt, msg.args)
File "/opt/has/conductor/conductor/controller/rpc.py", line 89, in plans_get
plans = self.Plan.query.get_plan_by_col('id', plan_id)
File "/opt/has/conductor/conductor/common/music/model/search.py", line 87, in get_plan_by_col
rows = db_backend.DB_API.row_read(
File "/opt/has/conductor/conductor/common/etcd/api.py", line 130, in row_read
schema = self.get_value(f'{keyspace}/{table}')
File "/opt/has/conductor/conductor/common/etcd/api.py", line 75, in get_value
raw_value = self.get_raw_value(key)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 72, in get_raw_value
return self.get_client().get(key)[0]
File "/opt/has/conductor/conductor/common/etcd/api.py", line 69, in get_client
raise EtcdClientException("Failed to establish connection with ETCD. GRPC {}".format(rpc_error.code()))
conductor.common.etcd.utils.EtcdClientException: Failed to establish connection with ETCD. GRPC StatusCode.UNAVAILABLE
2022-04-15 02:40:16,467||140488413350688|INFO|component|conductor.common.music.messaging.component: [-] Message 08c0d04f-d7c1-4835-9e45-b888340225f0 method plans_get, status: error
2022-04-15 02:40:23,365||140488413350688|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to error, atomic update response from MUSIC SUCCESS
2022-04-15 02:40:48,087||140488445438752|INFO|component|conductor.common.music.messaging.component: [-] Message aba53457-28b6-470b-a97e-1132cd526c5b method plans_get received
2022-04-15 02:40:56,781||140488445438752|INFO|component|conductor.common.music.messaging.component: [-] Message aba53457-28b6-470b-a97e-1132cd526c5b method plans_get, status: completed
2022-04-15 02:41:01,787||140488445438752|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:41:48,683||140488400620320|INFO|component|conductor.common.music.messaging.component: [-] Message 3170c050-6ec9-4efc-87c5-0bc173d628c3 method plans_get received
2022-04-15 02:41:56,883||140488400620320|INFO|component|conductor.common.music.messaging.component: [-] Message 3170c050-6ec9-4efc-87c5-0bc173d628c3 method plans_get, status: completed
2022-04-15 02:42:04,287||140488400620320|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:43:27,355||140488431385376|INFO|component|conductor.common.music.messaging.component: [-] Message e053340f-48e0-4060-9fe7-ba53d8809603 method plans_get received
2022-04-15 02:43:41,255||140488431385376|INFO|component|conductor.common.music.messaging.component: [-] Message e053340f-48e0-4060-9fe7-ba53d8809603 method plans_get, status: completed
2022-04-15 02:43:48,881||140488431385376|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:45:23,389||140488396376864|INFO|component|conductor.common.music.messaging.component: [-] Message 60e7a801-61be-4cd8-b352-2c3a528003fe method plans_get received
2022-04-15 02:45:33,383||140488396376864|INFO|component|conductor.common.music.messaging.component: [-] Message 60e7a801-61be-4cd8-b352-2c3a528003fe method plans_get, status: completed
2022-04-15 02:45:48,484||140488396376864|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:47:41,368||140488451803936|INFO|component|conductor.common.music.messaging.component: [-] Message 20ab69ec-6edd-4117-a5a8-86d1b19f611c method plans_get received
2022-04-15 02:47:49,659||140488451803936|INFO|component|conductor.common.music.messaging.component: [-] Message 20ab69ec-6edd-4117-a5a8-86d1b19f611c method plans_get, status: completed
2022-04-15 02:47:53,456||140488451803936|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:50:37,089||140488393194272|INFO|component|conductor.common.music.messaging.component: [-] Message 8a613b52-f951-41fc-a94b-88b04fbbff6c method plans_get received
2022-04-15 02:50:52,153||140488393194272|INFO|component|conductor.common.music.messaging.component: [-] Message 8a613b52-f951-41fc-a94b-88b04fbbff6c method plans_get, status: completed
2022-04-15 02:51:05,983||140488393194272|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:53:19,488||140488380463904|INFO|component|conductor.common.music.messaging.component: [-] Message be244e87-a726-49f8-9e08-a92dbfeded63 method plans_get received
2022-04-15 02:53:27,263||140488380463904|INFO|component|conductor.common.music.messaging.component: [-] Message be244e87-a726-49f8-9e08-a92dbfeded63 method plans_get, status: completed
2022-04-15 02:53:34,388||140488380463904|INFO|component|conductor.common.music.messaging.component: [-] updating the message status from working to completed, atomic update response from MUSIC SUCCESS
2022-04-15 02:55:14,388||140488452864800|ERROR|_utils|cotyledon._utils: [-] Unhandled exception
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/etcd/api.py", line 60, in get_client
return etcd3.client(host=self.host, port=self.port,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 1178, in client
return Etcd3Client(host=host,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 148, in __init__
resp = self.auth_stub.Authenticate(auth_request, self.timeout)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "etcdserver: request timed out"
debug_error_string = "{"created":"@1649991314.386916448","description":"Error received from peer ipv4:10.233.71.210:2379","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"etcdserver: request timed out","grpc_status":14}"
>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/cotyledon/_utils.py", line 95, in exit_on_exception
yield
File "/usr/local/lib/python3.9/site-packages/cotyledon/_service.py", line 139, in _run
self.run()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 477, in run
fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 440, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/futurist/_utils.py", line 49, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 334, in __check_for_messages
self._do()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 341, in _do
msgs = self.RPC.query.all()
File "/opt/has/conductor/conductor/common/music/model/search.py", line 80, in all
rows = db_backend.DB_API.row_read(**kwargs)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 136, in row_read
return self.get_values_prefix(key_prefix, pk_name, pk_value)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 82, in get_values_prefix
for kv in self.get_client().get_prefix(key_prefix)}
File "/opt/has/conductor/conductor/common/etcd/api.py", line 69, in get_client
raise EtcdClientException("Failed to establish connection with ETCD. GRPC {}".format(rpc_error.code()))
conductor.common.etcd.utils.EtcdClientException: Failed to establish connection with ETCD. GRPC StatusCode.UNAVAILABLE
2022-04-15 02:55:14,400||140488454986528|INFO|_service_manager|cotyledon._service_manager: [-] Child 6349 exited with status 2
2022-04-15 03:41:02,046||140488452864800|ERROR|_utils|cotyledon._utils: [-] Unhandled exception
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/etcd/api.py", line 60, in get_client
return etcd3.client(host=self.host, port=self.port,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 1178, in client
return Etcd3Client(host=host,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 148, in __init__
resp = self.auth_stub.Authenticate(auth_request, self.timeout)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "etcdserver: request timed out"
debug_error_string = "{"created":"@1649994062.044686901","description":"Error received from peer ipv4:10.233.74.59:2379","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"etcdserver: request timed out","grpc_status":14}"
>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/cotyledon/_utils.py", line 95, in exit_on_exception
yield
File "/usr/local/lib/python3.9/site-packages/cotyledon/_service.py", line 139, in _run
self.run()
File "/opt/has/conductor/conductor/controller/translator_svc.py", line 256, in run
fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 440, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/futurist/_utils.py", line 49, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/has/conductor/conductor/controller/translator_svc.py", line 202, in __check_for_templates
translating_plans = self.Plan.query.get_plan_by_col("status", self.Plan.TRANSLATING)
File "/opt/has/conductor/conductor/common/music/model/search.py", line 87, in get_plan_by_col
rows = db_backend.DB_API.row_read(
File "/opt/has/conductor/conductor/common/etcd/api.py", line 130, in row_read
schema = self.get_value(f'{keyspace}/{table}')
File "/opt/has/conductor/conductor/common/etcd/api.py", line 75, in get_value
raw_value = self.get_raw_value(key)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 72, in get_raw_value
return self.get_client().get(key)[0]
File "/opt/has/conductor/conductor/common/etcd/api.py", line 69, in get_client
raise EtcdClientException("Failed to establish connection with ETCD. GRPC {}".format(rpc_error.code()))
conductor.common.etcd.utils.EtcdClientException: Failed to establish connection with ETCD. GRPC StatusCode.UNAVAILABLE
2022-04-15 03:41:02,057||140488454986528|INFO|_service_manager|cotyledon._service_manager: [-] Child 212 exited with status 2
2022-04-15 03:49:43,749||140488452864800|ERROR|_utils|cotyledon._utils: [-] Unhandled exception
Traceback (most recent call last):
File "/opt/has/conductor/conductor/common/etcd/api.py", line 60, in get_client
return etcd3.client(host=self.host, port=self.port,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 1178, in client
return Etcd3Client(host=host,
File "/usr/local/lib/python3.9/site-packages/etcd3/client.py", line 148, in __init__
resp = self.auth_stub.Authenticate(auth_request, self.timeout)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "etcdserver: request timed out"
debug_error_string = "{"created":"@1649994583.748479826","description":"Error received from peer ipv4:10.233.74.59:2379","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"etcdserver: request timed out","grpc_status":14}"
>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/cotyledon/_utils.py", line 95, in exit_on_exception
yield
File "/usr/local/lib/python3.9/site-packages/cotyledon/_service.py", line 139, in _run
self.run()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 477, in run
fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 440, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/futurist/_utils.py", line 49, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 334, in __check_for_messages
self._do()
File "/opt/has/conductor/conductor/common/music/messaging/component.py", line 341, in _do
msgs = self.RPC.query.all()
File "/opt/has/conductor/conductor/common/music/model/search.py", line 80, in all
rows = db_backend.DB_API.row_read(**kwargs)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 130, in row_read
schema = self.get_value(f'{keyspace}/{table}')
File "/opt/has/conductor/conductor/common/etcd/api.py", line 75, in get_value
raw_value = self.get_raw_value(key)
File "/opt/has/conductor/conductor/common/etcd/api.py", line 72, in get_raw_value
return self.get_client().get(key)[0]
File "/opt/has/conductor/conductor/common/etcd/api.py", line 69, in get_client
raise EtcdClientException("Failed to establish connection with ETCD. GRPC {}".format(rpc_error.code()))
conductor.common.etcd.utils.EtcdClientException: Failed to establish connection with ETCD. GRPC StatusCode.UNAVAILABLE
2022-04-15 03:49:43,761||140488454986528|INFO|_service_manager|cotyledon._service_manager: [-] Child 11071 exited with status 2