Commit d95a587d authored by David Hendriks's avatar David Hendriks
Browse files

repaired bugs, fixed the free_store_memaddr, worked on slum & condor

parent 26484c5d
Loading
Loading
Loading
Loading
+268 −152

File changed.

Preview size limit exceeded, changes collapsed.

+34 −38
Original line number Diff line number Diff line
@@ -16,6 +16,8 @@ grid_options_defaults_dict = {
    "tmp_dir": temp_dir(),  # Setting the temp dir of the program
    "main_pid": -1,  # Placeholder for the main process id of the run.
    # "output_dir":
    "commandline_input": "",

    ##########################
    # Execution log:
    ##########################
@@ -118,55 +120,48 @@ grid_options_defaults_dict = {
    ########################################
    # Slurm stuff
    ########################################
    # slurm_ntasks=>1, # 1 CPU required per job
    # slurm_partition=>'all', # MUST be defined
    # slurm_jobname=>'binary_grid', # not required but useful
    # slurm_use_all_node_CPUs=>0, # if given nodes, set to 1
    #                             # if given CPUs, set to 0

    "slurm": 0,  # dont use the slurm by default. 1 = use slurm
    "slurm_command": "",  # Command that slurm runs (e.g. run_flexigrid or join_datafiles)
    "slurm_dir": "",  # working directory containin scripts output logs etc.
    "slurm_ntasks": 1, # CPUs required per array job: usually only need this
    "slurm_command": "",  # Command that slurm runs (e.g. evolve or join_datafiles)
    "slurm_dir": "",  # working directory containing scripts output logs etc.
    "slurm_njobs": 0, # number of scripts; set to 0 as default
    "slurm_jobid": '', # slurm job id (%A)
    "slurm_memory": 512, # in MB, the memory use of the job
    "slurm_warn_max_memory": 1024, # in MB : warn if mem req. > this
    "slurm_use_all_node_CPUs": 0, # 1 = use all of a node's CPUs. 0 = use a given amount of CPUs
    "slurm_postpone_join": 0, # if 1 do not join on slurm, join elsewhere

    # slurm_jobarrayindex=>'', # slurm job array index (%a)
    #     slurm_jobname=>'binary_grid', # set to binary_grid
    #     slurm_postpone_join=>0, # if 1, data is not joined, e.g. if you
    # # want to do it off the slurm grid (e.g. with more RAM)
    #     slurm_postpone_sbatch=>0, # if 1, don't submit, just make the script
    # # (defaults to $ENV{PWD} if undef)
    #     slurm_partition=>undef,
    #     slurm_ntasks=>1, # 1 CPU required per array job: usually only need this
    #     slurm_time=>0, # 0 = infinite time
    # # you will want to use this if your Slurm SelectType is e.g. linear
    # # which means it allocates all the CPUs in a node to the job
    # slurm_control_CPUs=>0, # if so, leave this many for Perl control (0)
    #     slurm_array=>undef,# override for --array, useful for rerunning jobs
    "slurm_postpone_join": 0, # if 1 do not join on slurm, join elsewhere. want to do it off the slurm grid (e.g. with more RAM)
    "slurm_jobarrayindex": '', # slurm job array index (%a)
    "slurm_jobname": 'binary_grid', # default
    "slurm_partition": None,
    "slurm_time": 0, # total time. 0 = infinite time
    "slurm_postpone_sbatch": 0, # if 1: don't submit, just make the script
    "slurm_array": None,# override for --array, useful for rerunning jobs
    "slurm_use_all_node_CPUs": 0, # if given nodes, set to 1
                                # if given CPUs, set to 0
    # you will want to use this if your Slurm SelectType is e.g. linear
    # which means it allocates all the CPUs in a node to the job
    "slurm_control_CPUs": 0, # if so, leave this many for Pythons control (0)
    "slurm_array": None, # override for --array, useful for rerunning jobs
    "slurm_partition": None, # MUST be defined
    "slurm_extra_settings": {}, # Place to put extra configuration for the SLURM batch file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so. 


    ########################################
    # Condor stuff
    ########################################
    # condor=>0, # 1 to use condor, 0 otherwise
    #     condor_command=>'',# condor command e.g. "run_flexigrid",
    # # "join_datafiles"
    # condor_dir=>'', # working directory containing e.g.
    # # scripts, output, logs (e.g. should be NFS available to all)
    # condor_njobs=>'', # number of scripts
    # condor_jobid=>'', # condor job id
    # condor_postpone_join=>0, # if 1, data is not joined, e.g. if you
    # # want to do it off the condor grid (e.g. with more RAM)
    # condor_join_machine=>undef, # if defined then this is the machine on which the join command
    # should be launched (must be sshable and not postponed)
    # condor_join_pwd=>undef, # directory the join should be in
    # # (defaults to $ENV{PWD} if undef)
    # condor_memory=>1024, # in MB, the memory use (ImageSize) of the job
    # condor_universe=>'vanilla', # usually vanilla universe
    "condor": 0, # 1 to use condor, 0 otherwise
    "condor_command": '', # condor command e.g. "evolve", "join"
    "condor_dir": '', # working directory containing e.g. scripts, output, logs (e.g. should be NFS available to all)
    "condor_njobs": '', # number of scripts/jobs that CONDOR will run in total
    "condor_jobid": '', # condor job id
    "condor_postpone_join": 0, # if 1, data is not joined, e.g. if you want to do it off the condor grid (e.g. with more RAM)
    # "condor_join_machine": None, # if defined then this is the machine on which the join command should be launched (must be sshable and not postponed)
    "condor_join_pwd": '', # directory the join should be in (defaults to $ENV{PWD} if undef)
    "condor_memory": 1024, # in MB, the memory use (ImageSize) of the job
    "condor_universe": 'vanilla', # usually vanilla universe
    "condor_extra_settings": {}, # Place to put extra configuration for the CONDOR submit file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so. 

    # snapshots and checkpoints
    # condor_snapshot_on_kill=>0, # if 1 snapshot on SIGKILL before exit
    # condor_load_from_snapshot=>0, # if 1 check for snapshot .sv file and load it if found
    # condor_checkpoint_interval=>0, # checkpoint interval (seconds)
@@ -185,6 +180,7 @@ grid_options_defaults_dict = {
    # condor_resubmit_submitted=>0,
    # condor_resubmit_running=>0,
    # condor_resubmit_crashed=>0,

    ##########################
    # Unordered. Need to go through this. Copied from the perl implementation.
    ##########################
+1 −1
Original line number Diff line number Diff line
@@ -71,7 +71,7 @@ int free_store_memaddr(long int store_memaddr,
#define NO_OUTPUT

#ifdef BINARY_C_PYTHON_DEBUG
  #define debug_printf(fmt, ...)  printf(fmt, __VA_ARGS__);
  #define debug_printf(fmt, ...)  printf(fmt, ##__VA_ARGS__);
#else
  #define debug_printf(fmt, ...)    /* Do nothing */
#endif
+6 −3
Original line number Diff line number Diff line
@@ -506,12 +506,14 @@ static PyObject* binary_c_free_persistent_data_memaddr_and_return_json_output(Py
static PyObject* binary_c_free_store_memaddr(PyObject *self, PyObject *args)
{
    /* Python binding that calls the c function that free's the store memory */

    long int store_memaddr = -1;

    /* Parse the input tuple */
    if(!PyArg_ParseTuple(args, "l", &store_memaddr))
    {
        // printf("Error: got a bad input\n");
        fprintf(stderr,
                "Error (in function: binary_c_free_store_memaddr): Got a bad input\n");
        return NULL;
    }

@@ -530,13 +532,14 @@ static PyObject* binary_c_free_store_memaddr(PyObject *self, PyObject *args)

    if(error_buffer != NULL && strlen(error_buffer)>0)
    {
        // printf("Error (in function: binary_c_free_store_memaddr): %s", error_buffer);
        fprintf(stderr,
                "Error (in function: binary_c_free_store): %s\n",
                "Error (in function: binary_c_free_store_memaddr): %s\n",
                error_buffer);
    }
    
    Safe_free(buffer);
    Safe_free(error_buffer);

    return 0;
    return Py_BuildValue("");
}
+2 −0
Original line number Diff line number Diff line
@@ -522,6 +522,7 @@ int free_store_memaddr(long int store_memaddr,
    {
        // load the store from the integer that has been passed
        store = (void*)store_memaddr;
        debug_printf("Took long int store_memaddr %ld and loaded it to %p\n", store_memaddr, (void*)&store);
    }
    else
    {
@@ -538,6 +539,7 @@ int free_store_memaddr(long int store_memaddr,
                        -1                  // argc
    );

    printf("freed store memaddr\n");
    /* output to strings */
    stardata->preferences->internal_buffering = INTERNAL_BUFFERING_STORE;
    stardata->preferences->batchmode = BATCHMODE_LIBRARY;
Loading