Fixed ensemble issues (fdaf033e) · Commits · binary_c / binary_c-python

binarycpython/tests/c_bindings/test_ensemble.py

+47 −123

Original line number	Diff line number	Diff line
		@@ -140,93 +140,6 @@ def pickle_n_checksize(obj):
		print("size: {}mb".format(file_stats.st_size / (1024 * 1024)))
		os.remove(name)


		def test_full_ensemble_output_new():
		"""
		Function to just output the whole ensemble
		"""

		argstring_1 = return_argstring(defer_ensemble=0, ensemble_filters_off=0)
		print(argstring_1)
		quit()
		output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
		# print(output_1)
		ensemble_jsons_1 = [
		line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
		]

		print("start")
		start = time.time()
		json_1 = eval(ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :])

		# json_1 = eval(
		# [line for line in _binary_c_bindings.run_system(argstring=argstring_1).splitlines() if line.startswith("ENSEMBLE_JSON")][0][len("ENSEMBLE_JSON ") :]
		# )
		stop = time.time()
		print("stop")

		pickle_n_checksize(json_1)

		print("took {}s to decode".format(stop - start))
		# print("Size of the json in memory: {}".format(sys.getsizeof(json_1)))

		print(json_1.keys())
		# assert statements:
		assert "number_counts" in json_1.keys()
		assert "HRD" in json_1.keys()
		assert "HRD(t)" in json_1.keys()
		assert "Xyield" in json_1.keys()
		assert "distributions" in json_1.keys()
		assert "scalars" in json_1.keys()


		def test_full_ensemble_output():
		"""
		Function to just output the whole ensemble
		"""

		argstring_1 = return_argstring(defer_ensemble=0, ensemble_filters_off=1)

		# print(argstring_1)
		# quit()
		argstring_1 += " ensemble_filter_MERGED 1 "
		argstring_1 += " ensemble_filter_ORBIT 1 "
		argstring_1 += " ensemble_filter_SCALARS 1 "
		argstring_1 += " ensemble_filter_CHEMICALLY_PECULIAR 1 "
		argstring_1 += " ensemble_filter_SPECTRAL_TYPES 1 "
		argstring_1 += " ensemble_filter_HRD 1 "

		output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
		pickle_n_checksize(output_1)
		ensemble_jsons_1 = [
		line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
		]

		pickle_n_checksize(ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :])

		print("start")
		start = time.time()
		json_1 = handle_ensemble_string_to_json(
		ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :]
		)
		stop = time.time()
		print("stop")

		pickle_n_checksize(json_1)

		print("took {}s to decode".format(stop - start))
		# print("Size of the json in memory: {}".format(sys.getsizeof(json_1)))

		# print(json_1)
		# assert statements:
		assert "number_counts" in json_1.keys()
		assert "HRD" in json_1.keys()
		assert "HRD(t)" in json_1.keys()
		assert "Xyield" in json_1.keys()
		assert "distributions" in json_1.keys()
		assert "scalars" in json_1.keys()


		def test_adding_ensemble_output():
		"""
		Function that adds the output of 2 ensembles and compares it to the output that we get by deferring the first output
		@@ -377,27 +290,6 @@ def test_adding_ensemble_output():
		), assert_message_1
		# assert inspect_dict(test_1_merged_dict, print_structure=False) == inspect_dict(test_3_json, print_structure=False), assert_message_2


		def test_combine_with_empty_json():
		"""
		Test for merging with an empty dict
		"""

		argstring_1 = return_argstring(defer_ensemble=0)
		output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
		ensemble_jsons_1 = [
		line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
		]
		json_1 = handle_ensemble_string_to_json(
		ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :]
		)

		assert_message = (
		"combining output json with empty dict should give same result as initial json"
		)
		assert merge_dicts(json_1, {}) == json_1, assert_message


		def test_free_and_json_output():
		"""
		Function that tests the freeing of the memory adress and the output of the json
		@@ -415,37 +307,69 @@ def test_free_and_json_output():
		persistent_data_memaddr = _binary_c_bindings.return_persistent_data_memaddr()

		# Evolve and defer output
		print("evolving")
		output_1_deferred = _binary_c_bindings.run_system(
		argstring=argstring_1, persistent_data_memaddr=persistent_data_memaddr
		)
		print("Evolved")
		print("Output:")
		print(textwrap.indent(str(output_1_deferred), "\t"))

		# Free memory adress
		print("freeing")
		json_output_by_freeing = _binary_c_bindings.free_persistent_data_memaddr_and_return_json_output(
		persistent_data_memaddr
		)
		print("Freed")
		print("Output:")
		print(textwrap.indent(str(json_output_by_freeing), "\t"))
		# print(textwrap.indent(str(json_output_by_freeing), "\t"))

		parsed_json = handle_ensemble_string_to_json(
		json_output_by_freeing.splitlines()[0][len("ENSEMBLE_JSON ") :],
		)
		print(parsed_json)

		# ensemble_jsons_1 = [line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")]
		# json_1 = json.loads(ensemble_jsons_1[0][len("ENSEMBLE_JSON "):], cls=binarycDecoder)
		assert "number_counts" in parsed_json.keys(), "Output not correct"

		def test_combine_with_empty_json():
		"""
		Test for merging with an empty dict
		"""

		argstring_1 = return_argstring(defer_ensemble=0)
		output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
		ensemble_jsons_1 = [
		line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
		]
		json_1 = handle_ensemble_string_to_json(
		ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :]
		)


		assert_message = (
		"combining output json with empty dict should give same result as initial json"
		)
		assert merge_dicts(json_1, {}) == json_1, assert_message

		#############
		def test_full_ensemble_output():
		"""
		Function to just output the whole ensemble
		"""

		argstring_1 = return_argstring(defer_ensemble=0, ensemble_filters_off=0)
		# print(argstring_1)

		output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
		ensemble_jsons_1 = [
		line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
		]
		json_1 = handle_ensemble_string_to_json(ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :])

		# assert statements:
		assert "number_counts" in json_1.keys()
		assert "HRD" in json_1.keys()
		assert "HRD(t)" in json_1.keys()
		# assert "Xyield" in json_1.keys()
		assert "distributions" in json_1.keys()
		assert "scalars" in json_1.keys()

		def all():
		# test_return_persistent_data_memaddr()
		# test_passing_persistent_data_to_run_system()
		test_full_ensemble_output_new()
		# test_full_ensemble_output()
		test_return_persistent_data_memaddr()
		test_passing_persistent_data_to_run_system()
		test_full_ensemble_output()
		test_adding_ensemble_output()
		test_free_and_json_output()
		test_combine_with_empty_json()

binarycpython/utils/functions.py

+5 −13

Original line number	Diff line number	Diff line
		@@ -975,16 +975,8 @@ def handle_ensemble_string_to_json(raw_output):
		Function that deals with the raw output of the ensemble and
		creates a working JSON dictionary out of it.

		It seems that JSON does not like strings that contain dictionaries which have a number as a key, but more specifically it is the fact that binary-c outputs as 5e2 instead of

		e.g. with python 3.6.4: json.loads("{5e2: 10}") leads to a JSONDecodeError. To fix this, I do the following:

		with ensemble_output being the literal string of output, and json_output the correct jsonified version of that.

		eval_dict = ast.literal_eval(ensemble_output)
		json_output = json.loads(json.dumps(eval_dict), cls=binarycDecoder)

		Note: this does pose somewhat of a risk, since eval'ing something might lead to unexpected behaviour.
		Having this wrapper makes it easy to
		"""

		return json.loads(json.dumps(ast.literal_eval(raw_output)), cls=binarycDecoder)
		# return json.loads(json.dumps(ast.literal_eval(raw_output)), cls=binarycDecoder)
		return json.loads(raw_output, cls=binarycDecoder)