making the tests for persistent data actually useful. putting the ensemble functions in the gridls (da3dccd0) · Commits · binary_c / binary_c-python

binarycpython/utils/functions.py

+57 −11

Original line number	Diff line number	Diff line
		@@ -8,6 +8,9 @@ useful functions for the user
		import json
		import os
		import tempfile
		import copy
		import inspect

		from collections import defaultdict

		import h5py
		@@ -19,15 +22,25 @@ import binary_c_python_api
		# utility functions
		########################################################

		def remove_file(file, verbose=0):
		def verbose_print(message, verbosity, minimal_verbosity):
		"""
		Function that decides whether to print a message based on the current verbosity
		and its minimum verbosity

		if verbosity is equal or higher than the minimum, then we print
		"""

		if verbosity >= minimal_verbosity:
		print(message)

		def remove_file(file, verbosity=0):
		"""
		Function to remove files but with verbosity
		"""

		if os.path.exists(file):
		try:
		if verbose > 0:
		print("Removed {}".format(file))
		verbose_print("Removed {}".format(file), verbosity, 1)
		os.remove(file)

		except FileNotFoundError as inst:
		@@ -116,6 +129,18 @@ def create_hdf5(data_dir, name):
		# version_info functions
		########################################################

		def return_binary_c_version_info(parsed=False):
		"""
		Function that returns the version information of binary_c
		"""

		version_info = binary_c_python_api.return_version_info().strip()

		if parsed:
		version_info = parse_binary_c_version_info(version_info)

		return version_info

		def parse_binary_c_version_info(version_info_string):
		"""
		Function that parses the binary_c version info. Length function with a lot of branches
		@@ -666,7 +691,7 @@ def load_logfile(logfile):
		# Ensemble dict functions
		########################################################

		def inspect_dict(dict_1, indent=0):
		def inspect_dict(dict_1, indent=0, print_structure=True):
		"""
		Function to inspect a dict.

		@@ -675,10 +700,16 @@ def inspect_dict(dict_1, indent=0):
		Prints out keys and their value types
		"""

		structure_dict = {}

		for key, value in dict_1.items():
		structure_dict[key] = type(value)
		if print_structure:
		print("\t"*indent, key, type(value))
		if isinstance(value, dict):
		inspect_dict(value, indent=indent+1)
		structure_dict[key] = inspect_dict(value, indent=indent+1)
		return structure_dict


		def merge_dicts(dict_1, dict_2):
		"""
		@@ -715,24 +746,27 @@ def merge_dicts(dict_1, dict_2):
		if isinstance(dict_1[key], (float, int)):
		new_dict[key] = dict_1[key]
		else:
		new_dict[key] = dict_1[key].deepcopy()
		copy_dict = copy.deepcopy(dict_1[key])
		new_dict[key] = copy_dict

		for key in unique_to_dict_2:
		if isinstance(dict_2[key], (float, int)):
		new_dict[key] = dict_2[key]
		else:
		new_dict[key] = dict_2[key].deepcopy()
		copy_dict = copy.deepcopy(dict_2[key])
		new_dict[key] = copy_dict

		# Go over the common keys:
		for key in overlapping_keys:
		# See whether the types are actually similar
		# See whether the types are actually the same
		if not type(dict_1[key]) is type(dict_2[key]):
		print("Error {} and {} are not of the same type and cannot be merged".format(
		dict_1[key], dict_2[key]))
		raise ValueError

		# TODO: Create a matrix of combinations here.
		# TODO: Could maybe be more compact
		# Here we check for the cases that we want to explicitly catch. Ints will be added,
		# floats will be added, lists will be appended (though that might change) and dicts will be
		# dealt with by calling this function again.
		else:
		# ints
		if isinstance(dict_1[key], int) and isinstance(dict_2[key], int):
		@@ -756,6 +790,7 @@ def merge_dicts(dict_1, dict_2):
		#
		return new_dict


		class binarycDecoder(json.JSONDecoder):
		"""
		Custom decoder to transform the numbers that are strings to actual floats
		@@ -788,3 +823,14 @@ class binarycDecoder(json.JSONDecoder):
		else:
		return o

		def binaryc_json_serializer(obj):
		"""
		Custom serializer for binary_c to use when functions are present in the dictionary
		that we want to export.

		Function objects will be turned into str representations of themselves
		"""

		if inspect.isfunction(obj):
		return str(obj)
		return obj

binarycpython/utils/grid.py

+142 −93

Original line number	Diff line number	Diff line
		@@ -34,12 +34,16 @@ from binarycpython.utils.functions import (
		remove_file,
		filter_arg_dict,
		get_help_all,
		return_binary_c_version_info,
		binaryc_json_serializer,
		verbose_print,
		binarycDecoder,
		merge_dicts
		)

		import binary_c_python_api


		# Todo-list
		# Tasks
		# TODO: add functionality to 'on-init' set arguments
		# TODO: add functionality to return the initial_abundance_hash
		# TODO: add functionality to return the isotope_hash
		@@ -81,6 +85,10 @@ class Population:
		# Set main process id
		self.grid_options["main_pid"] = os.getpid()

		# Set some memory dicts
		self.persistent_data_memory_dict = {}


		###################################################
		# Argument functions
		###################################################
		@@ -118,20 +126,21 @@ class Population:
		in the self.grid_options

		If neither of above is met; the key and the value get stored in a custom_options dict.

		TODO: catch those parameter names that contain an %d
		"""

		for key in kwargs:
		# Filter out keys for the bse_options
		if key in self.defaults.keys():
		if self.grid_options["verbose"] > 0:
		print("adding: {}={} to BSE_options".format(key, kwargs[key]))
		verbose_print("adding: {}={} to BSE_options".format(key, kwargs[key]), self.grid_options["verbose"], 1)
		self.bse_options[key] = kwargs[key]

		# Filter out keys for the grid_options
		elif key in self.grid_options.keys():
		if self.grid_options["verbose"] > 0:
		print("adding: {}={} to grid_options".format(key, kwargs[key]))
		verbose_print("adding: {}={} to grid_options".format(key, kwargs[key]), self.grid_options["verbose"], 1)
		self.grid_options[key] = kwargs[key]

		# The of the keys go into a custom_options dict
		else:
		print(
		@@ -159,8 +168,8 @@ class Population:
		# How its set up now is that as input you need to give --cmdline "metallicity=0.002"
		# Its checked if this exists and handled accordingly.
		if args.cmdline:
		if self.grid_options["verbose"] > 0:
		print("Found cmdline args. Parsing them now")
		verbose_print("Found cmdline args. Parsing them now", self.grid_options["verbose"], 1)

		# Grab the input and split them up, while accepting only non-empty entries
		cmdline_args = args.cmdline
		split_args = [
		@@ -276,8 +285,7 @@ class Population:

		# Load it into the grid_options
		self.grid_options["grid_variables"][grid_variable["name"]] = grid_variable
		if self.grid_options["verbose"] > 0:
		print("Added grid variable: {}".format(json.dumps(grid_variable, indent=4)))
		verbose_print("Added grid variable: {}".format(json.dumps(grid_variable, indent=4)), self.grid_options["verbose"], 1)

		###################################################
		# Return functions
		@@ -342,7 +350,7 @@ class Population:
		all_info["binary_c_defaults"] = binary_c_defaults

		if include_binary_c_version_info:
		binary_c_version_info = self.return_binary_c_version_info(parsed=True)
		binary_c_version_info = return_binary_c_version_info(parsed=True)
		all_info["binary_c_version_info"] = binary_c_version_info

		if include_binary_c_help_all:
		@@ -382,18 +390,18 @@ class Population:
		# Copy dict
		all_info_cleaned = copy.deepcopy(all_info)

		# Clean the all_info_dict: (i.e. transform the function objects to strings)
		if all_info_cleaned.get("population_settings", None):
		if all_info_cleaned["population_settings"]["grid_options"][
		"parse_function"
		]:
		all_info_cleaned["population_settings"]["grid_options"][
		"parse_function"
		] = str(
		all_info_cleaned["population_settings"]["grid_options"][
		"parse_function"
		]
		)
		# # Clean the all_info_dict: (i.e. transform the function objects to strings)
		# if all_info_cleaned.get("population_settings", None):
		# if all_info_cleaned["population_settings"]["grid_options"][
		# "parse_function"
		# ]:
		# all_info_cleaned["population_settings"]["grid_options"][
		# "parse_function"
		# ] = str(
		# all_info_cleaned["population_settings"]["grid_options"][
		# "parse_function"
		# ]
		# )

		if use_datadir:
		if not self.custom_options.get("base_filename", None):
		@@ -412,15 +420,15 @@ class Population:
		self.custom_options["data_dir"], settings_name
		)

		if self.grid_options["verbose"] > 0:
		print("Writing settings to {}".format(settings_fullname))
		verbose_print("Writing settings to {}".format(settings_fullname), self.grid_options["verbose"], 1)
		# if not outfile.endswith('json'):
		with open(settings_fullname, "w") as file:
		file.write(json.dumps(all_info_cleaned, indent=4))
		file.write(
		json.dumps(all_info_cleaned, indent=4, default=binaryc_json_serializer)
		)

		else:
		if self.grid_options["verbose"] > 0:
		print("Writing settings to {}".format(outfile))
		verbose_print("Writing settings to {}".format(outfile), self.grid_options["verbose"], 1)
		# if not outfile.endswith('json'):
		with open(outfile, "w") as file:
		file.write(json.dumps(all_info_cleaned, indent=4))
		@@ -432,9 +440,7 @@ class Population:
		"""

		# C_logging_code gets priority of C_autogen_code
		if self.grid_options["verbose"] > 0:
		print("Creating and loading custom logging functionality")

		verbose_print("Creating and loading custom logging functionality", self.grid_options["verbose"], 1)
		if self.grid_options["C_logging_code"]:
		# Generate entire shared lib code around logging lines
		custom_logging_code = binary_c_log_code(
		@@ -470,6 +476,53 @@ class Population:
		custom_logging_code, verbose=self.grid_options["verbose"]
		)

		###################################################
		# Ensemble functions
		###################################################

		def load_persistent_data_memory_dict(self):
		"""
		Function that loads a set amount (amt_cores) of persistent data memory adresses to
		pass to binary_c.

		TODO: fix the function
		"""

		for thread_nr in self.grid_options["amt_cores"]:
		persistent_data_memaddr = binary_c_python_api.binary_c_return_persistent_data_memaddr()
		self.persistent_data_memory_dict[thread_nr] = persistent_data_memaddr
		verbose_print("Created the following dict with persistent memaddresses: {}".format(self.persistent_data_memory_dict), self.grid_options["verbosity"], 1)

		def free_persistent_data_memory_and_combine_results(self):
		"""
		Function that loads a set amount of persisten data memory adresses to
		pass to binary_c.

		TODO: fix the function
		"""

		combined_ensemble_json = {}

		for key in self.persistent_data_memory_dict:
		persistent_data_memaddr = self.persistent_data_memory_dict[key]

		verbose_print("Freeing {} (thread {})and merging output to combined dict".format(persistent_data_memaddr, key), self.grid_options["verbosity"], 1)

		# Get the output and decode it correctly to get the numbers correct
		ensemble_json_output = binary_c_python_api.binary_c_free_persistent_data_memaddr_and_return_json_output(persistent_data_memaddr)
		parsed_json = json.loads(ensemble_json_output.splitlines()[0][len("ENSEMBLE_JSON "):], cls=binarycDecoder)

		# Combine the output with the main output
		combined_ensemble_json = merge_dicts(combined_ensemble_json, parsed_json)

		# Write results to file.
		# TODO: Make sure everything is checked beforehand
		full_output_filename = os.path.join(self.custom_options["data_dir"], self.custom_options["ensemble_output_name"])
		verbose_print("Writing ensemble output to {}".format(full_output_filename), self.grid_options["verbosity"], 1)




		###################################################
		# Evolution functions
		###################################################
		@@ -725,11 +778,10 @@ class Population:
		Function to test the evolution of a system. Calls the api binding directly.
		"""

		if self.grid_options["verbose"] > 0:
		print("running a single system as a test")
		verbose_print("running a single system as a test", self.grid_options["verbose"], 1)

		m1 = 15.0 # Msun
		m2 = 14.0 # Msun
		m_1 = 15.0 # Msun
		m_2 = 14.0 # Msun
		separation = 0 # 0 = ignored, use period
		orbital_period = 4530.0 # days
		eccentricity = 0.0
		@@ -737,8 +789,8 @@ class Population:
		max_evolution_time = 15000
		argstring = "binary_c M_1 {0:g} M_2 {1:g} separation {2:g} orbital_period {3:g}\
		eccentricity {4:g} metallicity {5:g} max_evolution_time {6:g}".format(
		m1,
		m2,
		m_1,
		m_2,
		separation,
		orbital_period,
		eccentricity,
		@@ -779,8 +831,7 @@ class Population:
		Results in a generated file that contains a system_generator function.
		"""

		if self.grid_options["verbose"] > 0:
		print("Generating grid code")
		verbose_print("Generating grid code", self.grid_options["verbose"], 1)

		# Some local values
		code_string = ""
		@@ -1143,8 +1194,7 @@ class Population:
		# Stop of code generation. Here the code is saved and written

		# Save the gridcode to the grid_options
		if self.grid_options["verbose"] > 0:
		print("Saving grid code to grid_options")
		verbose_print("Saving grid code to grid_options", self.grid_options["verbose"], 1)

		self.grid_options["code_string"] = code_string

		@@ -1154,8 +1204,7 @@ class Population:
		)
		self.grid_options["gridcode_filename"] = gridcode_filename

		if self.grid_options["verbose"] > 0:
		print("Writing grid code to {}".format(gridcode_filename))
		verbose_print("Writing grid code to {}".format(gridcode_filename), self.grid_options["verbose"], 1)

		with open(gridcode_filename, "w") as file:
		file.write(code_string)
		@@ -1167,13 +1216,11 @@ class Population:
		"""

		# Code to load the

		if self.grid_options["verbose"] > 0:
		print(
		"Loading grid code function from {}".format(
		verbose_print(
		message="Loading grid code function from {}".format(
		self.grid_options["gridcode_filename"]
		)
		)
		),
		verbosity=self.grid_options["verbose"], minimal_verbosity=1)

		spec = importlib.util.spec_from_file_location(
		"binary_c_python_grid",
		@@ -1185,8 +1232,7 @@ class Population:

		self.grid_options["system_generator"] = generator

		if self.grid_options["verbose"] > 0:
		print("Grid code loaded")
		verbose_print("Grid code loaded", self.grid_options["verbose"], 1)

		def dry_run(self):
		"""
		@@ -1349,8 +1395,8 @@ class Population:
		"""

		if evol_type == "single":
		if self.grid_options["verbose"] > 0:
		print("Cleaning up the custom logging stuff. type: single")
		verbose_print("Cleaning up the custom logging stuff. type: single", self.grid_options["verbose"], 1)

		# TODO: Unset custom logging code

		# TODO: Unset function memory adress
		@@ -1364,11 +1410,14 @@ class Population:
		)

		if evol_type == "population":
		if self.grid_options["verbose"] > 0:
		print("Cleaning up the custom logging stuffs. type: population")
		verbose_print("Cleaning up the custom logging stuffs. type: population", self.grid_options["verbose"], 1)

		# TODO: make sure that these also work. not fully sure if necessary tho.
		# whether its a single file, or a dict of files/memaddresses

		if evol_type == "MC":
		pass

		def increment_probtot(self, prob):
		"""
		Function to add to the total probability

tests/test_persistent_data.py

+88 −41

File changed.

Preview size limit exceeded, changes collapsed.