generate-submission-script-cpu-parallel.py

Go to the documentation of this file.

from itertools import combinations
import os
from pathlib import Path
 
# Configurable Options
application = "euler"
solver = "rusanov"
machine = "sapphire_rapid"
runtime_hours = 24
patch_sizes = [3, 6, 16]
numbers_of_threads = [1, 2, 4, 8, 16, 32, 64]
number_of_samples = 10
 
numbers_of_patches = {
    3: ["1", "2", "4", "8", "16", "32", "64", "128", "256", "512", "1024", "2048", "4096", "8192"],
    6: ["1", "2", "4", "8", "16", "32", "64", "128", "256", "512", "1024", "2048", "4096", "8192"],
    16: ["1", "2", "4", "8", "16", "32", "64", "128", "256", "512", "1024", "2048", "4096", "8192"]
}
 
partition = "dine2"
account = "do015"
set_environment = f"source {Path.home()}/set_env_x86_cpu_only.sh"
cores = 64
memory = "1TB"
 
# First stage of passes
#optional_passes = {
#    "taskloop": "--enable-omp-tasks --omp-task-version=taskloop",
#    "task": "--enable-omp-tasks --omp-task-version=task",
#    "taskloop_single-region": "--enable-omp-tasks --omp-task-version=taskloop --omp-single-region",
#    "task_single-region": "--enable-omp-tasks --omp-task-version=task --omp-single-region"
#}
 
# Second stage of passes
optional_passes = {
    "taskloop_single-region_memref-flattening": "--enable-memref-flattening --enable-omp-tasks --omp-task-version=taskloop --omp-single-region",
    "taskloop_single-region_function-inlining": "--enable-function-inlining --enable-omp-tasks --omp-task-version=taskloop --omp-single-region",
    "taskloop_single-region_memory-fusion": "--enable-memory-fusion --enable-omp-tasks --omp-task-version=taskloop --omp-single-region",
    "taskloop_single-region_loop-fusion": "--enable-loop-fusion --enable-omp-tasks --omp-task-version=taskloop --omp-single-region",
    "taskloop_single-region_perfect-loop-nesting": "--enable-imperfect-nested-loops-to-perfect --enable-omp-tasks --omp-task-version=taskloop --omp-single-region"
}
 
output_name = f"{application}_kernel_benchmark"
job_name = f"{application}_kernel_benchmark"
output_root = f"{Path.home()}/Peano/benchmarks/exahype2/kernel-benchmarks/"
script_name = f"submit_kernel_benchmark-cpu-parallel-{application}-{solver}.sh"
 
if solver == "fd4":
    benchmark_script_name = "kernel-benchmarks-fd4-dsl.py"
    benchmark_app_name = "KernelBenchmarksFD4DSL.Release"
if solver == "rusanov":
    benchmark_script_name = "kernel-benchmarks-fv-rusanov-dsl.py"
    benchmark_app_name = "KernelBenchmarksFVRusanovDSL.Release"
 
def generate_submission_script_fragment(use_cpp: bool, patch_size: int, number_of_patches: list[str], passes: list[str], output_directory: str):
    if use_cpp:
        print_statement = f"""echo "Patch Size: {patch_size}, C++" """
    else:
        print_statement = f"""echo "Patch Size: {patch_size}, MLIR, {", ".join(passes)}" """
    pass_flags = [optional_passes[enabled_pass] for enabled_pass in passes]
    return f"""{print_statement}
python {benchmark_script_name} -d 3 -cpu {"--enable-cpp" if use_cpp else "--enable-mlir"} -parallel -ps {patch_size} -p {" ".join(number_of_patches)} -samples {number_of_samples} {" ".join(pass_flags)}
for number_of_threads in "${{numbers_of_threads[@]}}"; do
    echo "Number of threads: ${{number_of_threads}}"
    export OMP_NUM_THREADS=${{number_of_threads}}
    ./{benchmark_app_name} > {output_directory}/patch_size_{patch_size}_threads_${{number_of_threads}}.txt
done
"""
compile_scripts = []
run_scripts = []
submission_scripts = []
 
for patch_size in patch_sizes:
    number_of_patches = numbers_of_patches[patch_size]
 
    pass_list = []
    for i in range(0, len(optional_passes) + 1):
        for combination in list(combinations(optional_passes, i)):
            pass_list.append(combination)
 
#    # CPP
#    output_directory = output_root + f"results/{application}/{solver}/cpu_parallel/{machine}/cpp"
#    if not os.path.exists(output_directory):
#        os.makedirs(output_directory)
#    submission_scripts.append(generate_submission_script_fragment(True, patch_size, number_of_patches, [], output_directory))
#    
#    # MLIR No Passes
#    output_directory = output_root + f"results/{application}/{solver}/cpu_parallel/{machine}/no-passes"
#    if not os.path.exists(output_directory):
#        os.makedirs(output_directory)
#    submission_scripts.append(generate_submission_script_fragment(False, patch_size, number_of_patches, [], output_directory))
 
    # MLIR Passes
    for optional_pass in optional_passes.keys():
        output_directory = output_root + f"results/{application}/{solver}/cpu_parallel/{machine}/{optional_pass}"
        if not os.path.exists(output_directory):
            os.makedirs(output_directory)
        submission_scripts.append(generate_submission_script_fragment(False, patch_size, number_of_patches, [optional_pass], output_directory))
 
with open(script_name, "w") as file:
    file.write(f"""#!/bin/bash -l
 
#SBATCH -o {output_name}.out
#SBATCH -e {output_name}.err
#SBATCH -t {runtime_hours}:00:00
#SBATCH --nodes=1
#SBATCH -J {job_name}
#SBATCH -c {cores}
#SBATCH -p {partition}
#SBATCH -A {account}
#SBATCH --mem={memory}
 
export PATH=~/Peano/src/exahype2/mlir/build/bin/:$PATH
export PYTHONPATH=~/Peano/python/:~/Peano/applications/exahype2/ccz4/:~/Peano/src/
{set_environment}
 
numbers_of_threads=({" ".join([str(number_of_threads) for number_of_threads in numbers_of_threads])})
""")
    for script in submission_scripts:
        file.write(script + "\n")