reformatted the code (2fd73538) · Commits · binary_c / binary_c-python

binarycpython/utils/functions.py

+41 −13

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ import binary_c_python_api
		# utility functions
		########################################################


		def verbose_print(message, verbosity, minimal_verbosity):
		"""
		Function that decides whether to print a message based on the current verbosity
		@@ -33,6 +34,7 @@ def verbose_print(message, verbosity, minimal_verbosity):
		if verbosity >= minimal_verbosity:
		print(message)


		def remove_file(file, verbosity=0):
		"""
		Function to remove files but with verbosity
		@@ -46,6 +48,7 @@ def remove_file(file, verbosity=0):
		except FileNotFoundError as inst:
		print("Error while deleting file {}: {}".format(file, inst))


		def temp_dir():
		"""
		Function to return the path the custom logging library shared object
		@@ -62,6 +65,7 @@ def temp_dir():

		return path


		def create_hdf5(data_dir, name):
		"""
		Function to create an hdf5 file from the contents of a directory:
		@@ -125,10 +129,12 @@ def create_hdf5(data_dir, name):

		hdf5_file.close()


		########################################################
		# version_info functions
		########################################################


		def return_binary_c_version_info(parsed=False):
		"""
		Function that returns the version information of binary_c
		@@ -141,6 +147,7 @@ def return_binary_c_version_info(parsed=False):

		return version_info


		def parse_binary_c_version_info(version_info_string):
		"""
		Function that parses the binary_c version info. Length function with a lot of branches
		@@ -234,16 +241,19 @@ def parse_binary_c_version_info(version_info_string):

		return version_info_dict


		########################################################
		# binary_c output functions
		########################################################


		def output_lines(output):
		"""
		Function that outputs the lines that were recieved from the binary_c run.
		"""
		return output.splitlines()


		def parse_output(output, selected_header):
		"""
		Function that parses output of binary_c:
		@@ -309,10 +319,12 @@ def parse_output(output, selected_header):

		return final_values_dict


		########################################################
		# Argument and default value functions
		########################################################


		def get_defaults(filter_values=False):
		"""
		Function that calls the binaryc get args function and cast it into a dictionary.
		@@ -335,6 +347,7 @@ def get_defaults(filter_values=False):

		return default_dict


		def get_arg_keys():
		"""
		Function that return the list of possible keys to give in the arg string
		@@ -342,6 +355,7 @@ def get_arg_keys():

		return get_defaults().keys()


		def filter_arg_dict(arg_dict):
		"""
		Function to filter out keys that contain values included in ['NULL', 'Function', '']
		@@ -357,6 +371,7 @@ def filter_arg_dict(arg_dict):

		return new_dict


		def create_arg_string(arg_dict, sort=False, filter_values=False):
		"""
		Function that creates the arg string for binary_c.
		@@ -377,10 +392,12 @@ def create_arg_string(arg_dict, sort=False, filter_values=False):
		arg_string = arg_string.strip()
		return arg_string


		########################################################
		# Help functions
		########################################################


		def get_help(param_name="", print_help=True, fail_silently=False):
		"""
		Function that returns the help info for a given parameter.
		@@ -479,6 +496,7 @@ def get_help(param_name="", print_help=True, fail_silently=False):
		)
		return None


		def get_help_all(print_help=True):
		"""
		Function that reads out the output of the help_all api call to binary_c
		@@ -585,6 +603,7 @@ def get_help_all(print_help=True):

		return help_all_dict


		def get_help_super(print_help=False, fail_silently=True):
		"""
		Function that first runs get_help_all, and then per argument also run
		@@ -612,9 +631,7 @@ def get_help_super(print_help=False, fail_silently=True):

		# Get detailed help info
		detailed_help = get_help(
		parameter_name,
		print_help=False,
		fail_silently=fail_silently,
		parameter_name, print_help=False, fail_silently=fail_silently,
		)

		if detailed_help:
		@@ -638,16 +655,17 @@ def get_help_super(print_help=False, fail_silently=True):

		section["parameters"][parameter_name] = parameter


		if print_help:
		print(json.dumps(help_all_super_dict, indent=4))

		return help_all_super_dict


		########################################################
		# logfile functions
		########################################################


		def load_logfile(logfile):
		"""
		Experimental function that parses the generated logfile of binary_c.
		@@ -687,10 +705,12 @@ def load_logfile(logfile):

		print(event_list)


		########################################################
		# Ensemble dict functions
		########################################################


		def inspect_dict(dict_1, indent=0, print_structure=True):
		"""
		Function to inspect a dict.
		@@ -760,8 +780,11 @@ def merge_dicts(dict_1, dict_2):
		for key in overlapping_keys:
		# See whether the types are actually the same
		if not type(dict_1[key]) is type(dict_2[key]):
		print("Error {} and {} are not of the same type and cannot be merged".format(
		dict_1[key], dict_2[key]))
		print(
		"Error {} and {} are not of the same type and cannot be merged".format(
		dict_1[key], dict_2[key]
		)
		)
		raise ValueError

		# Here we check for the cases that we want to explicitly catch. Ints will be added,
		@@ -785,8 +808,11 @@ def merge_dicts(dict_1, dict_2):
		new_dict[key] = merge_dicts(dict_1[key], dict_2[key])

		else:
		print("Object types {},{} not supported".format(
		type(dict_1[key]), type(dict_2[key])))
		print(
		"Object types {},{} not supported".format(
		type(dict_1[key]), type(dict_2[key])
		)
		)
		#
		return new_dict

		@@ -797,7 +823,9 @@ class binarycDecoder(json.JSONDecoder):
		"""

		def decode(self, s):
		result = super().decode(s) # result = super(Decoder, self).decode(s) for Python 2.x
		result = super().decode(
		s
		) # result = super(Decoder, self).decode(s) for Python 2.x
		return self._decode(result)

		def _decode(self, o):
		@@ -808,7 +836,6 @@ class binarycDecoder(json.JSONDecoder):
		The try except might be a somewhat rough solution but it catches all cases.
		"""


		# Check if we can turn it into a float
		# if isinstance(o, str) or isinstance(o, unicode):
		if isinstance(o, str):
		@@ -823,6 +850,7 @@ class binarycDecoder(json.JSONDecoder):
		else:
		return o


		def binaryc_json_serializer(obj):
		"""
		Custom serializer for binary_c to use when functions are present in the dictionary

binarycpython/utils/grid.py

+163 −77

Original line number	Diff line number	Diff line
		@@ -38,7 +38,7 @@ from binarycpython.utils.functions import (
		binaryc_json_serializer,
		verbose_print,
		binarycDecoder,
		merge_dicts
		merge_dicts,
		)

		import binary_c_python_api
		@@ -88,7 +88,6 @@ class Population:
		# Set some memory dicts
		self.persistent_data_memory_dict = {}


		###################################################
		# Argument functions
		###################################################
		@@ -132,23 +131,44 @@ class Population:
		"""

		# Select the params that end with %d
		special_params = [el for el in list(self.defaults.keys()) if el.endswith('%d')]
		special_params = [el for el in list(self.defaults.keys()) if el.endswith("%d")]

		# Go over all the input
		for key in kwargs:
		# Filter out keys for the bse_options
		if key in self.defaults.keys():
		verbose_print("adding: {}={} to BSE_options".format(key, kwargs[key]), self.grid_options["verbosity"], 1)
		verbose_print(
		"adding: {}={} to BSE_options".format(key, kwargs[key]),
		self.grid_options["verbosity"],
		1,
		)
		self.bse_options[key] = kwargs[key]

		# Extra check to check if the key fits one of parameter names that end with %d
		elif any([True if (key.startswith(param[:-2]) and len(param[:-2]) < len(key)) else False for param in special_params]):
		verbose_print("adding: {}={} to BSE_options by catching the %d".format(key, kwargs[key]), self.grid_options["verbosity"], 1)
		elif any(
		[
		True
		if (key.startswith(param[:-2]) and len(param[:-2]) < len(key))
		else False
		for param in special_params
		]
		):
		verbose_print(
		"adding: {}={} to BSE_options by catching the %d".format(
		key, kwargs[key]
		),
		self.grid_options["verbosity"],
		1,
		)
		self.bse_options[key] = kwargs[key]

		# Filter out keys for the grid_options
		elif key in self.grid_options.keys():
		verbose_print("adding: {}={} to grid_options".format(key, kwargs[key]), self.grid_options["verbosity"], 1)
		verbose_print(
		"adding: {}={} to grid_options".format(key, kwargs[key]),
		self.grid_options["verbosity"],
		1,
		)
		self.grid_options[key] = kwargs[key]

		# The of the keys go into a custom_options dict
		@@ -178,7 +198,11 @@ class Population:
		# How its set up now is that as input you need to give --cmdline "metallicity=0.002"
		# Its checked if this exists and handled accordingly.
		if args.cmdline:
		verbose_print("Found cmdline args. Parsing them now", self.grid_options["verbosity"], 1)
		verbose_print(
		"Found cmdline args. Parsing them now",
		self.grid_options["verbosity"],
		1,
		)

		# Grab the input and split them up, while accepting only non-empty entries
		cmdline_args = args.cmdline
		@@ -295,7 +319,11 @@ class Population:

		# Load it into the grid_options
		self.grid_options["grid_variables"][grid_variable["name"]] = grid_variable
		verbose_print("Added grid variable: {}".format(json.dumps(grid_variable, indent=4)), self.grid_options["verbosity"], 1)
		verbose_print(
		"Added grid variable: {}".format(json.dumps(grid_variable, indent=4)),
		self.grid_options["verbosity"],
		1,
		)

		###################################################
		# Return functions
		@@ -419,18 +447,32 @@ class Population:
		self.custom_options["data_dir"], settings_name
		)

		verbose_print("Writing settings to {}".format(settings_fullname), self.grid_options["verbosity"], 1)
		verbose_print(
		"Writing settings to {}".format(settings_fullname),
		self.grid_options["verbosity"],
		1,
		)
		# if not outfile.endswith('json'):
		with open(settings_fullname, "w") as file:
		file.write(
		json.dumps(all_info_cleaned, indent=4, default=binaryc_json_serializer)
		json.dumps(
		all_info_cleaned, indent=4, default=binaryc_json_serializer
		)
		)

		else:
		verbose_print("Writing settings to {}".format(outfile), self.grid_options["verbosity"], 1)
		verbose_print(
		"Writing settings to {}".format(outfile),
		self.grid_options["verbosity"],
		1,
		)
		# if not outfile.endswith('json'):
		with open(outfile, "w") as file:
		file.write(json.dumps(all_info_cleaned, indent=4, default=binaryc_json_serializer))
		file.write(
		json.dumps(
		all_info_cleaned, indent=4, default=binaryc_json_serializer
		)
		)

		def set_custom_logging(self):
		"""
		@@ -439,7 +481,11 @@ class Population:
		"""

		# C_logging_code gets priority of C_autogen_code
		verbose_print("Creating and loading custom logging functionality", self.grid_options["verbosity"], 1)
		verbose_print(
		"Creating and loading custom logging functionality",
		self.grid_options["verbosity"],
		1,
		)
		if self.grid_options["C_logging_code"]:
		# Generate entire shared lib code around logging lines
		custom_logging_code = binary_c_log_code(
		@@ -486,9 +532,17 @@ class Population:
		"""

		for thread_nr in self.grid_options["amt_cores"]:
		persistent_data_memaddr = binary_c_python_api.binary_c_return_persistent_data_memaddr()
		persistent_data_memaddr = (
		binary_c_python_api.binary_c_return_persistent_data_memaddr()
		)
		self.persistent_data_memory_dict[thread_nr] = persistent_data_memaddr
		verbose_print("Created the following dict with persistent memaddresses: {}".format(self.persistent_data_memory_dict), self.grid_options["verbosity"], 1)
		verbose_print(
		"Created the following dict with persistent memaddresses: {}".format(
		self.persistent_data_memory_dict
		),
		self.grid_options["verbosity"],
		1,
		)

		def free_persistent_data_memory_and_combine_results_and_output(self):
		"""
		@@ -503,22 +557,39 @@ class Population:
		for key in self.persistent_data_memory_dict:
		persistent_data_memaddr = self.persistent_data_memory_dict[key]

		verbose_print("Freeing {} (thread {})and merging output to combined dict".format(persistent_data_memaddr, key), self.grid_options["verbosity"], 1)
		verbose_print(
		"Freeing {} (thread {})and merging output to combined dict".format(
		persistent_data_memaddr, key
		),
		self.grid_options["verbosity"],
		1,
		)

		# Get the output and decode it correctly to get the numbers correct
		ensemble_json_output = binary_c_python_api.binary_c_free_persistent_data_memaddr_and_return_json_output(persistent_data_memaddr)
		parsed_json = json.loads(ensemble_json_output.splitlines()[0][len("ENSEMBLE_JSON "):], cls=binarycDecoder)
		ensemble_json_output = binary_c_python_api.binary_c_free_persistent_data_memaddr_and_return_json_output(
		persistent_data_memaddr
		)
		parsed_json = json.loads(
		ensemble_json_output.splitlines()[0][len("ENSEMBLE_JSON ") :],
		cls=binarycDecoder,
		)

		# Combine the output with the main output
		combined_ensemble_json = merge_dicts(combined_ensemble_json, parsed_json)

		# Write results to file.
		# TODO: Make sure everything is checked beforehand
		full_output_filename = os.path.join(self.custom_options["data_dir"], self.custom_options["ensemble_output_name"])
		verbose_print("Writing ensemble output to {}".format(full_output_filename), self.grid_options["verbosity"], 1)
		full_output_filename = os.path.join(
		self.custom_options["data_dir"], self.custom_options["ensemble_output_name"]
		)
		verbose_print(
		"Writing ensemble output to {}".format(full_output_filename),
		self.grid_options["verbosity"],
		1,
		)

		# Output to dir:
		with open(full_output_filename, 'w') as output_file:
		with open(full_output_filename, "w") as output_file:
		output_file.write(json.dumps(combined_ensemble_json, indent=4))

		###################################################
		@@ -559,7 +630,11 @@ class Population:
		## check the settings:
		if self.bse_options["ensemble"] == 1:
		if not self.bse_options["ensemble_defer"] == 1:
		verbose_print("Error, if you want to run an ensemble in a population, the output needs to be deferred", self.grid_options["verbosity"], 0)
		verbose_print(
		"Error, if you want to run an ensemble in a population, the output needs to be deferred",
		self.grid_options["verbosity"],
		0,
		)
		raise ValueError

		#######################
		@@ -628,7 +703,9 @@ class Population:
		# Unload functions

		# Unload store
		binary_c_python_api.binary_c_free_store_memaddr(self.grid_options["store_memaddr"])
		binary_c_python_api.binary_c_free_store_memaddr(
		self.grid_options["store_memaddr"]
		)

		def evolve_system_mp(self, binary_cmdline_string):
		"""
		@@ -677,7 +754,7 @@ class Population:

		# Get argument line
		argline = self.return_argline(self.bse_options)
		verbose_print("Running {}".format(argline), self.grid_options['verbosity'], 1)
		verbose_print("Running {}".format(argline), self.grid_options["verbosity"], 1)

		# Run system
		out = binary_c_python_api.run_system(
		@@ -1185,7 +1262,9 @@ class Population:
		# Stop of code generation. Here the code is saved and written

		# Save the gridcode to the grid_options
		verbose_print("Saving grid code to grid_options", self.grid_options["verbosity"], 1)
		verbose_print(
		"Saving grid code to grid_options", self.grid_options["verbosity"], 1
		)

		self.grid_options["code_string"] = code_string

		@@ -1195,7 +1274,11 @@ class Population:
		)
		self.grid_options["gridcode_filename"] = gridcode_filename

		verbose_print("Writing grid code to {}".format(gridcode_filename), self.grid_options["verbosity"], 1)
		verbose_print(
		"Writing grid code to {}".format(gridcode_filename),
		self.grid_options["verbosity"],
		1,
		)

		with open(gridcode_filename, "w") as file:
		file.write(code_string)
		@@ -1211,7 +1294,9 @@ class Population:
		message="Loading grid code function from {}".format(
		self.grid_options["gridcode_filename"]
		),
		verbosity=self.grid_options["verbosity"], minimal_verbosity=1)
		verbosity=self.grid_options["verbosity"],
		minimal_verbosity=1,
		)

		spec = importlib.util.spec_from_file_location(
		"binary_c_python_grid",
		@@ -1386,7 +1471,11 @@ class Population:
		"""

		if evol_type == "single":
		verbose_print("Cleaning up the custom logging stuff. type: single", self.grid_options["verbosity"], 1)
		verbose_print(
		"Cleaning up the custom logging stuff. type: single",
		self.grid_options["verbosity"],
		1,
		)

		# TODO: Unset custom logging code

		@@ -1401,7 +1490,11 @@ class Population:
		)

		if evol_type == "population":
		verbose_print("Cleaning up the custom logging stuffs. type: population", self.grid_options["verbosity"], 1)
		verbose_print(
		"Cleaning up the custom logging stuffs. type: population",
		self.grid_options["verbosity"],
		1,
		)

		# TODO: make sure that these also work. not fully sure if necessary tho.
		# whether its a single file, or a dict of files/memaddresses
		@@ -1422,7 +1515,6 @@ class Population:
		"""
		self.grid_options["count"] += 1


		def set_loggers(self):
		"""
		Function to set the loggers for the execution of the grid
		@@ -1435,7 +1527,7 @@ class Population:
		os.makedirs(os.path.dirname(binary_c_logfile), exist_ok=True)

		# Set up logger
		self.logger = logging.getLogger('binary_c_python_logger')
		self.logger = logging.getLogger("binary_c_python_logger")
		self.logger.setLevel(self.grid_options["verbosity"])

		# Reset handlers
		@@ -1443,14 +1535,12 @@ class Population:

		# Set formatting of output
		log_formatter = logging.Formatter(
		'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
		"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
		)

		# Make and add filehandlers
		# make handler for output to file
		handler_file = logging.FileHandler(
		filename=os.path.join(binary_c_logfile)
		)
		handler_file = logging.FileHandler(filename=os.path.join(binary_c_logfile))
		handler_file.setFormatter(log_formatter)
		handler_file.setLevel(logging.INFO)

		@@ -1463,10 +1553,6 @@ class Population:
		self.logger.addHandler(handler_file)
		self.logger.addHandler(handler_stdout)





		# def join_result_dicts(self):
		# """
		# Function to join the result dictionaries

binarycpython/utils/grid_options_defaults.py

+10 −8

Original line number	Diff line number	Diff line
		@@ -20,7 +20,9 @@ grid_options_defaults_dict = {
		# Execution log:
		##########################
		"verbosity": 0, # Level of verbosity of the simulation. 0=INFO,
		"log_file": os.path.join(temp_dir(), 'binary_c_python.log'), # Set to None to not log to file. The directory will be created
		"log_file": os.path.join(
		temp_dir(), "binary_c_python.log"
		), # Set to None to not log to file. The directory will be created
		##########################
		# binary_c files
		##########################
		@@ -67,7 +69,7 @@ grid_options_defaults_dict = {
		"population_type": "grid", #
		"population_type_options": [
		"grid",
		], # TODO: fill later with monte carlo, source file etc
		], # Available choices for type of population generation # TODO: fill later with monte carlo, source file etc
		"count": 0, # total count of systems
		"probtot": 0, # total probability
		"weight": 1.0, # weighting for the probability
		@@ -82,7 +84,7 @@ grid_options_defaults_dict = {
		"modulo": 1, # run modulo n of the grid.
		## Grid type evolution
		"grid_variables": {}, # grid variables
		"grid_code": None, # literal grid code
		"grid_code": None, # literal grid code: contains the whole script that'll be written to a file
		"gridcode_filename": None, # filename of gridcode
		## Monte carlo type evolution
		# TODO: make MC options
		@@ -135,9 +137,9 @@ grid_options_defaults_dict = {
		# # which means it allocates all the CPUs in a node to the job
		# slurm_control_CPUs=>0, # if so, leave this many for Perl control (0)
		# slurm_array=>undef,# override for --array, useful for rerunning jobs
		# ########################################
		# # Condor stuff
		# ########################################
		########################################
		# Condor stuff
		########################################
		# condor=>0, # 1 to use condor, 0 otherwise
		# condor_command=>'',# condor command e.g. "run_flexigrid",
		# # "join_datafiles"

binarycpython/utils/plot_functions.py

+108 −86

File changed.

Preview size limit exceeded, changes collapsed.

setup.py

+11 −7

Original line number	Diff line number	Diff line
		@@ -16,21 +16,26 @@ def readme():
		with open("README.md") as file:
		return file.read()


		def license():
		"""Opens license file and returns the content"""
		with open("LICENSE.md") as file:
		return file.read()


		def check_version(installed_binary_c_version, required_binary_c_versions):
		"""Function to check the installed version and compare it to the required version"""
		message = """
		The binary_c version that is installed ({}) does not match the binary_c versions ({})
		that this release of the binary_c python module requires.
		""".format(installed_binary_c_version, required_binary_c_versions)
		""".format(
		installed_binary_c_version, required_binary_c_versions
		)
		assert installed_binary_c_version in required_binary_c_versions, message


		###
		REQUIRED_BINARY_C_VERSIONS = ['2.1.7']
		REQUIRED_BINARY_C_VERSIONS = ["2.1.7"]

		####
		GSL_DIR = os.getenv("GSL_DIR", None)
		@@ -55,9 +60,7 @@ CWD = os.getcwd()
		BINARY_C_CONFIG = os.path.join(BINARY_C_DIR, "binary_c-config")

		BINARY_C_VERSION = (
		subprocess.run(
		[BINARY_C_CONFIG, "version"], stdout=subprocess.PIPE, check=True
		)
		subprocess.run([BINARY_C_CONFIG, "version"], stdout=subprocess.PIPE, check=True)
		.stdout.decode("utf-8")
		.split()
		)
		@@ -189,8 +192,9 @@ setup(
		It is tested and designed to work for versions {}, we can't guarantee proper functioning for other versions

		If you want to use a different version of binary_c, download and install a different version of this package
		""".format(str(REQUIRED_BINARY_C_VERSIONS), str(REQUIRED_BINARY_C_VERSIONS)),

		""".format(
		str(REQUIRED_BINARY_C_VERSIONS), str(REQUIRED_BINARY_C_VERSIONS)
		),
		author="David Hendriks, Robert Izzard and Jeff Andrews",
		author_email="davidhendriks93@gmail.com/d.hendriks@surrey.ac.uk,\
		r.izzard@surrey.ac.uk/rob.izzard@gmail.com andrews@physics.uoc.gr",