hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 10:31:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Redirect worker stdout/stderr to log files. (#646)

Browse source 
* local scheduler

* redirect output files to be associated with workers rather than the local scheduler

* fixed formatting

* fixes

* Moved output redirection logic to worker.py.

* Changed write mode.

* Fixed formatting.

* Added comment.

* Reuse log file creation in services.py.

* Fix linting.

* Fix problem in which multiple processes attempt to create /tmp/raylogs at the same time.
This commit is contained in:
alanamarzoev 2017-06-08 18:30:48 -07:00 • committed by Robert Nishihara
parent fff50d824c
commit ee1d4e5ea2
2 changed files with 17 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
python/ray/services.py
View file

@ -1145,7 +1145,13 @@ def new_log_files(name, redirect_output):
logs_dir = "/tmp/raylogs"
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
try:
os.makedirs(logs_dir)
except OSError as e:
if e.errno != os.errno.EEXIST:
raise e
print("Attempted to create '/tmp/raylogs', but the directory already "
"exists.")
# Change the log directory permissions so others can use it. This is
# important when multiple people are using the same machine.
os.chmod(logs_dir, 0o0777)

10
python/ray/worker.py
View file

@ -1233,6 +1233,14 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
worker.actor_id = actor_id
worker.connected = True
worker.set_mode(mode)
# Redirect worker output and error to their own files.
if mode == WORKER_MODE:
log_stdout_file, log_stderr_file = services.new_log_files("worker", True)
sys.stdout = log_stdout_file
sys.stderr = log_stderr_file
services.record_log_files_in_redis(info["redis_address"],
info["node_ip_address"],
[log_stdout_file, log_stderr_file])
# The worker.events field is used to aggregate logging information and
# display it in the web UI. Note that Python lists protected by the GIL,
# which is important because we will append to this field from multiple
@ -1275,6 +1283,8 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
worker.redis_client.hmset(
b"Workers:" + worker.worker_id,
{"node_ip_address": worker.node_ip_address,
"stdout_file": os.path.abspath(log_stdout_file.name),
"stderr_file": os.path.abspath(log_stderr_file.name),
"plasma_store_socket": info["store_socket_name"],
"plasma_manager_socket": info["manager_socket_name"],
"local_scheduler_socket": info["local_scheduler_socket_name"]})