fixing data generation

Author: Dpbm

Makefile

@@ -4,25 +4,25 @@ TARGET_PATH ?= "."
 clean-all: clean-dataset clean-pred clean-ghz clean-model clean-checkpoints clean-history clean-gen-checkpoint
 
 clean-dataset:
-	rm -rf $(TARGET_PATH)/dataset/ $(TARGET_PATH)/dataset.csv $(TARGET_PATH)/*.h5 $(TARGET_PATH)/dataset-images.zip
+	sudo rm -rf $(TARGET_PATH)/dataset/ $(TARGET_PATH)/dataset.csv $(TARGET_PATH)/*.h5 $(TARGET_PATH)/dataset-images.zip
 
 clean-pred:
-	rm -rf $(TARGET_PATH)/ghz-prediction.pth
+	sudo rm -rf $(TARGET_PATH)/ghz-prediction.pth
 
 clean-ghz:
-	rm -rf $(TARGET_PATH)/ghz.pth $(TARGET_PATH)/ghz.jpeg
+	sudo rm -rf $(TARGET_PATH)/ghz.pth $(TARGET_PATH)/ghz.jpeg
 
 clean-model:
-	rm -rf $(TARGET_PATH)/model_*
+	sudo rm -rf $(TARGET_PATH)/model_*
 
 clean-checkpoints:
-	rm -rf $(TARGET_PATH)/checkpoint_*
+	sudo rm -rf $(TARGET_PATH)/checkpoint_*
 
 clean-gen-checkpoint:
-	rm -rf $(TARGET_PATH)/gen_checkpoint.json
+	sudo rm -rf $(TARGET_PATH)/gen_checkpoint.json
 
 clean-history:
-	rm -rf $(TARGET_PATH)/history.json
+	sudo rm -rf $(TARGET_PATH)/history.json
 
 pack:
 	zip -r $(TARGET_PATH)/dataset-images.zip $(TARGET_PATH)/dataset/

dags/dataset.py

@@ -17,6 +17,7 @@
     remove_duplicated_files,
     transform_images,
     start_df,
+    shuffle_csv,
     Checkpoint,
     Stages,
 )
@@ -42,6 +43,7 @@
 GEN_IMAGES_TASK_ID = "gen_images"
 REMOVE_DUPLICATES_TASK_ID = "remove_duplicates"
 TRANSFORM_TASK_ID = "transform_images"
+SHUFFLE_DATASET_ID = "shuffle_df"
 
 
 def next_step(checkpoint: Checkpoint) -> str:
@@ -55,6 +57,9 @@ def next_step(checkpoint: Checkpoint) -> str:
     if checkpoint.stage == Stages.GEN_IMAGES:
         return GEN_IMAGES_TASK_ID
 
+    if checkpoint.stage == Stages.SHUFFLE:
+        return SHUFFLE_DATASET_ID
+
     if checkpoint.stage == Stages.DUPLICATES:
         return REMOVE_DUPLICATES_TASK_ID
 
@@ -131,14 +136,35 @@ def update_checkpoint(checkpoint: Checkpoint, stage: Stages):
     Qiskit framework.
     """
 
-    transtion_gen_to_remove = PythonOperator(
-        task_id="gen_to_remove",
+    transtion_gen_to_shuffle = PythonOperator(
+        task_id="gen_to_shuffle",
+        python_callable=update_checkpoint,
+        op_args=[checkpoint, Stages.SHUFFLE],
+    )
+
+    transtion_gen_to_shuffle.doc_md = """
+    Update checkpoint to start shuffling rows.
+    """
+
+    shuffle = PythonOperator(
+        task_id=SHUFFLE_DATASET_ID,
+        python_callable=shuffle_csv,
+        op_args=[folder],
+        trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS
+    )
+
+    shuffle.doc_md = """
+    Shuffle dataset rows.
+    """
+    
+    transtion_shuffle_to_remove = PythonOperator(
+        task_id="shuffle_to_remove",
         python_callable=update_checkpoint,
         op_args=[checkpoint, Stages.DUPLICATES],
     )
 
-    transtion_gen_to_remove.doc_md = """
-    Update checkpoint to start removing duplicated files.
+    transtion_shuffle_to_remove.doc_md = """
+    Update checkpoint to start deleting duplicated rows.
     """
 
     remove_duplicates = PythonOperator(
@@ -228,11 +254,14 @@ def update_checkpoint(checkpoint: Checkpoint, stage: Stages):
     gen_df >> branch_checkpoint
 
     branch_checkpoint >> gen_images
+    branch_checkpoint >> shuffle
     branch_checkpoint >> remove_duplicates
     branch_checkpoint >> transform_img
 
-    gen_images >> transtion_gen_to_remove
-    transtion_gen_to_remove >> remove_duplicates
+    gen_images >> transtion_gen_to_shuffle
+    transtion_gen_to_shuffle >> shuffle
+    shuffle >> transtion_shuffle_to_remove
+    transtion_shuffle_to_remove >> remove_duplicates
     remove_duplicates >> transition_remove_to_transform
     transition_remove_to_transform >> transform_img
     transform_img >> reset_checkpoint

dataset.py

@@ -31,6 +31,7 @@
     images_h5_file,
     images_gen_checkpoint_file,
     dataset_file_tmp,
+    SCALE_CIRCUIT_SIZE
 )
 from utils.datatypes import FilePath, df_schema, Dimensions
 from utils.image import transform_image
@@ -49,6 +50,7 @@ class Stages(Enum):
     """Enum for dataset generation stages"""
 
     GEN_IMAGES = "gen"
+    SHUFFLE = "shuffle"
     DUPLICATES = "duplicates"
     TRANSFORM = "transform"
 
@@ -200,7 +202,7 @@ def generate_circuit_images(
         qc_copy.add_register(classical_register)
         qc_copy.measure(measurement, list(range(total_measurements)))
 
-        drawing = qc_copy.draw("mpl", filename=image_path)
+        drawing = qc_copy.draw("mpl", filename=image_path, fold=-1, scale=SCALE_CIRCUIT_SIZE)
         plt.close(drawing)
         del drawing
 
@@ -405,6 +407,22 @@ def get_duplicated_files_list_by_diff(
 
     return duplicated_files.to_list()  # type: ignore
 
+def shuffle_df(df:pl.DataFrame) -> pl.DataFrame:
+    """
+    Shffle DF rows to ensure no sequential logic is kept.
+    """
+    return df.sample(fraction=1.0, shuffle=True, seed=32)
+
+
+def shuffle_csv(target_folder:FilePath):
+    """
+    Shffle CSV rows.
+    """
+    print("%sShuffling DF....%s"%(Colors.GREENBG,Colors.ENDC))
+    file_path = dataset_file(target_folder)
+    df = pl.read_csv(file_path)
+    df = shuffle_df(df)
+    df.write_csv(file_path)
 
 def transform_images(
     target_folder: FilePath, new_dim: Dimensions, checkpoint: Checkpoint
@@ -434,7 +452,7 @@ def transform_images(
         with h5py.File(images_h5_file(target_folder), "a") as file:
             for image_path in tqdm(collected_rows):
                 with Image.open(image_path) as img:
-                    tensor = transform_image(img, max_width, max_height)
+                    tensor = transform_image(img, max_width)
                     file.create_dataset(f"{image_i}", data=tensor)
 
                 image_i += 1
@@ -522,8 +540,15 @@ def main(args: Arguments):
         checkpoint.stage = Stages.DUPLICATES
         checkpoint.index = 0
 
+    if checkpoint.stage == Stages.SHUFFLE:
+        shuffle_df()
+
+        checkpoint.stage = Stages.DUPLICATES
+        checkpoint.index = 0
+
     if checkpoint.stage == Stages.DUPLICATES:
         remove_duplicated_files(args.target_folder, checkpoint)
+
         checkpoint.stage = Stages.TRANSFORM
         checkpoint.index = 0
 

generate/fix_default_img_sizes.py

@@ -0,0 +1,73 @@
+import sys
+import os
+from typing import Optional, Tuple
+from pathlib import Path
+import re
+
+from PIL import Image
+    
+base_path = str(Path(__file__).resolve().parent.parent)
+sys.path.append(base_path)
+    
+import matplotlib.pyplot as plt
+from qiskit import QuantumCircuit
+
+from utils.constants import DEFAULT_MAX_TOTAL_GATES, DEFAULT_NUM_QUBITS
+from random_circuit import generate_circuit, generate_circuit_worst_case
+
+CONSTANTS_FILE_PATH = os.path.join(base_path, "utils", "constants.py")
+TESTS_FILE_PATH = os.path.join(base_path, "tests")
+SCALE_SIZE = 0.5
+
+def draw(qc:QuantumCircuit, ax:Optional[plt.Axes]=None) -> plt.Figure:
+    """Draw circuit with tweaks."""
+    return qc.draw("mpl", scale=SCALE_SIZE, fold=-1, ax=ax)
+
+if __name__ == "__main__":
+    print("Testing the min size of a circuit image...")
+    qc1 = generate_circuit(DEFAULT_NUM_QUBITS, 0)
+    small_fig_path = os.path.join(TESTS_FILE_PATH, "small.png")
+    img1 = draw(qc1)
+    img1.savefig(small_fig_path, bbox_inches="tight")
+    plt.close(img1)
+    min_width,min_height = Image.open(small_fig_path).size
+    print(f"Size -> {min_width}x{min_height}px")
+
+    print("Testing the max size of a circuit image...")
+    qc2 = generate_circuit_worst_case(DEFAULT_NUM_QUBITS, DEFAULT_MAX_TOTAL_GATES)
+    large_fig_path = os.path.join(TESTS_FILE_PATH, "large.png")
+    img2 = draw(qc2)
+    img2.savefig(large_fig_path, bbox_inches="tight")
+    plt.close(img2)
+    max_width,max_height = Image.open(large_fig_path).size
+    print(f"Size -> {max_width}x{max_height}px")
+
+    # fig,ax = plt.subplots(2)
+    # draw(qc1,ax[0])
+    # draw(qc2,ax[1])
+    # plt.show()
+    
+
+    with open(CONSTANTS_FILE_PATH, "r+", encoding="utf-8") as file:
+        read = file.read()
+
+        pattern_dim = r'DEFAULT_NEW_DIM *= *\( *[0-9]{2,} *, *[0-9]{2,} *\)'
+        pattern_scale = r'SCALE_CIRCUIT_SIZE *= *0\.[1-9]'
+
+        new_dim_data = f"DEFAULT_NEW_DIM = ({max_width}, {max_height})"
+        new_scale_factor = f"SCALE_CIRCUIT_SIZE = {SCALE_SIZE}"
+
+        new_data = read
+
+        if re.search(pattern_dim, new_data):
+            new_data = re.sub(pattern_dim, new_dim_data, new_data)
+        else:
+            new_data += new_dim_data
+
+        if re.search(pattern_scale, new_data):
+            new_data = re.sub(pattern_scale, new_scale_factor, new_data)
+        else:
+            new_data += new_scale_factor
+    
+    with open(CONSTANTS_FILE_PATH, "w", encoding="utf-8") as file:
+        file.write(new_data)

generate/random_circuit.py

@@ -60,12 +60,8 @@ def get_random_gate(cls) -> QiskitGate:
         gate = random.choice(cls.gates)
         return gate()
 
-
-def get_random_circuit(n_qubits: int, total_gates: int) -> QuantumCircuit:
-    """Generate a random circuit based on the amount of qubits and gates."""
-
-    total_gates = random.randint(0, total_gates)
-
+def generate_circuit(n_qubits:int, total_gates:int) -> QuantumCircuit:
+    """Generate a circuit based on the amount of gates"""
     qc = QuantumCircuit(n_qubits)
 
     for _ in range(total_gates):
@@ -85,3 +81,21 @@ def get_random_circuit(n_qubits: int, total_gates: int) -> QuantumCircuit:
             qc.barrier()
 
     return qc
+
+def generate_circuit_worst_case(n_qubits: int, total_gates: int) -> QuantumCircuit:
+    """Generate the longes circuit as possible"""
+    qc = QuantumCircuit(n_qubits)
+    
+    for _ in range(total_gates):
+        gate = SingleQubitGate.get_random_gate()
+        qc.append(gate, [0])
+        qc.barrier()
+
+    return qc
+
+def get_random_circuit(n_qubits: int, total_gates: int) -> QuantumCircuit:
+    """Generate a random circuit based on the amount of qubits and gates."""
+
+    total_gates = random.randint(0, total_gates)
+    return generate_circuit(n_qubits,total_gates)
+

ghz.py

@@ -6,7 +6,7 @@
 
 from utils.image import transform_image
 from args.parser import parse_args
-from utils.constants import ghz_image_file, ghz_file
+from utils.constants import ghz_image_file, ghz_file, SCALE_CIRCUIT_SIZE
 from utils.datatypes import FilePath, Dimensions
 
 
@@ -20,14 +20,13 @@ def gen_circuit(n_qubits: int, target_folder: FilePath, new_dim: Dimensions):
     qc.measure_all()
 
     ghz_image_path = ghz_image_file(target_folder)
-    qc.draw("mpl", filename=ghz_image_path)
+    qc.draw("mpl", filename=ghz_image_path, fold=-1, scale=SCALE_CIRCUIT_SIZE)
 
     with Image.open(ghz_image_path) as file:
         width, height = new_dim
-        tensor = transform_image(file, width, height)
+        tensor = transform_image(file, width)
         torch.save(tensor, ghz_file(target_folder))
 
-
 if __name__ == "__main__":
     args = parse_args()
     gen_circuit(args.n_qubits, args.target_folder, args.new_image_dim)

tests/large.png

@@ -11,6 +11,7 @@
     save_df,
     start_df,
     get_duplicated_files_list_by_diff,
+    shuffle_df
 )
 from utils.datatypes import df_schema
 
@@ -233,6 +234,15 @@ def test_remove_duplicates_sequence(self, base_df, tmp_df, tmp_df2):
         assert os.path.exists(tmp_df)
         assert len(pl.read_csv(tmp_df)) == 8
 
+    def test_shuffle_df(self,base_df):
+        df = pl.read_csv(base_df)
+        no_shuffled = df["file"].to_list()
+        shuffled = shuffle_df(df)["file"].to_list()
+        print(no_shuffled, shuffled)
+
+        assert no_shuffled != shuffled
+        
+
     # SINCE SAVING A LAZY FRAME AS CSV IN THE SAME FILE IS NOT STABLE,
     # WE GONNA IGNORE THE TESTS BELLOW. FOR THE PRODUCTION CODE, WE GONNA
     # SAVE THE UPDATED LAZY FRAME IN A DIFFRERENT FILE AND THEN RENAME IT.

tests/small.png

@@ -0,0 +1,28 @@
+import os
+
+from PIL import Image
+
+from utils.image import transform_image
+from utils.constants import DEFAULT_NEW_DIM
+from train import Model
+
+IMAGES_TESTS_PATH = os.path.join(".", "tests")
+
+class TestModel:
+    """
+    Test the Model itself.
+    """
+
+    def test_image_on_model(self):
+        """Check if the images fit in the network."""
+        model = Model().half()
+
+        width,height = DEFAULT_NEW_DIM
+
+        small_img = Image.open(os.path.join(IMAGES_TESTS_PATH, "small.png"))
+        small_img_tr = transform_image(small_img, width)
+        model.forward(small_img_tr)
+        
+        large_img = Image.open(os.path.join(IMAGES_TESTS_PATH, "large.png"))
+        large_img_tr = transform_image(small_img, width)
+        model.forward(large_img_tr)