<jupyter_start><jupyter_text>This notebook is used to cleanly organize single-molecule tracking data by scanning through JSON parameter files used to generate figures and moving all XML TrackMate files into a specified folder, re-creating the JSON parameter file if needed. Optionally, it can also scan through the raw data file folder and move/organize all relevant raw data files.<jupyter_code># User-defined parameters

# str or list of str:
settings_files = ['../analysis_settings/Fig2F_HaloTagControls.json',
                '../analysis_settings/Fig3F_IRE1-HaloTag.json',
                '../analysis_settings/Fig4A_Tm_Tg_DTT.json',
                '../analysis_settings/Fig4B_IRE1mutants.json',
                '../analysis_settings/FigS1B_GST_dimer.json',
                '../analysis_settings/FigS2_ERstress_HaloControls.json',
                '../analysis_settings/FigS3D_IRE1-HaloTag_clones.json',
                '../analysis_settings/FigS4_Trajectory_density.json']

# Specify which data to organize (TrackMate outputs, raw data, or both)
#organize_tracks = True
organize_raw = True

# Specify source directories
# Reference directory to the relative paths in the settings JSON file(s)
base_dir_tracks = r'C:\Vlad\Work_sys_drive\Paper_data\Archive\processed'
# Directory in which all raw image files are found (subdirectories OK)
base_dir_raw = r'C:\Vlad\Work_sys_drive\Paper_data\raw\raw_unzipped'

#Specify target directories (absolute or relative to the notebook file)
final_dir_tracks = r'C:\Vlad\Work_sys_drive\Paper_code\Belyy_et_al_2021\data\processed'
final_dir_json = r'C:\Vlad\Work_sys_drive\Paper_code\Belyy_et_al_2021\analysis_settings\auto_gen'
final_dir_raw = r'C:\Vlad\Work_sys_drive\Paper_code\Belyy_et_al_2021\data\raw'

image_file_extension = 'tif' # for searching through image files
# Load modules
import json, os, copy, shutil, glob
from pathlib import Path

# Uncomment the following two lines for debugging
%load_ext autoreload
%autoreload 2

# Add source code directory (src) to path to enable user module import
module_dir = '../src'
os.sys.path.insert(0, module_dir)

# Import user modules from source code directory
#import parse_trackmate as pt
import correlation_analysis as corr
# Read and organize data


if type(settings_files) is not list: settings_files = [settings_files]

raw_files_copied = [] # to avoid duplicating raw files

for file in settings_files:
    print('Working on file: '+str(file))
    conditions, params = corr.read_analysis_params(file, print_summary=False)
    new_json = copy.deepcopy(params['raw_json'])
    origin_paths = []
    dest_paths = []
    for condition, paths in params['raw_json']['conditions'].items():
        if type(paths) is not list: paths = [paths]
        new_paths_rel = []
        for curr_path in paths:
            path = Path(curr_path)
            origin = (Path(base_dir_tracks) / path).resolve()
            origin_paths.append(origin)
            
            #Keep folder and parent folder
            dest_subfolder = origin.relative_to(origin.parents[1])
            dest = (Path(final_dir_tracks) / dest_subfolder).resolve()
            dest_paths.append(dest)
            
            #Determine relative path to put in the new JSON file
            json_relpath = Path(os.path.relpath(dest,final_dir_json))
            new_paths_rel.append(json_relpath.as_posix())
            
        new_json['conditions'][condition] = new_paths_rel
        
    # Move the xml and raw files without overwriting
    num_dirs = len(origin_paths)
    dir_counter = 1
    for origin, dest in zip(origin_paths, dest_paths):
        print('Working on directory '+str(dir_counter)+' out of '+str(num_dirs))
        print(str(origin))
        dir_counter = dir_counter + 1
        if not os.path.exists(dest): os.makedirs(dest)
        xml_files_in_origin = glob.glob(os.path.join(origin, '**/*.xml'), recursive=True)    
        if not xml_files_in_origin:
            print("WARNING: no valid xml files found in this directory!")
        else: print(str(len(xml_files_in_origin))+' xml files found here.')
        for file_origin in xml_files_in_origin:
            name = os.path.split(file_origin)[1]
            file_dest = os.path.join(dest, name)
            
            # Move xml track files
            if not os.path.exists(file_dest):
                shutil.copy(file_origin, dest)
            
            # Locate and move raw tif files if needed
            if organize_raw:
                # get source file name
                name_core = name[3:-4] # remove channel and extension
                search_string = '*'+name_core+'.'+image_file_extension
                # Look for matching tif file
                source_path = list(Path(base_dir_raw).rglob(search_string))
                if len(source_path) > 1:
                    print('Too many matches found! Problematic files:')
                    print(name_core)
                    for x in source_path: print(x)
                if len(source_path) == 0:
                    print('No matches found! Problematic file:')
                    print(name_core)
                    continue
                    
                source_path = source_path[0]
                
                if source_path in raw_files_copied:
                    continue # this file had already been moved
                raw_files_copied.append(source_path)
                
                #Organize folder and parent folder to match those of the xml file
                dest_subfolder = dest.relative_to(dest.parents[1])
                dest_folder = os.path.join(final_dir_raw, dest_subfolder)
                dest_folder = os.path.abspath(dest_folder)
                if not os.path.exists(dest_folder): os.makedirs(dest_folder)
                dest_file = os.path.join(dest_folder, os.path.split(source_path)[1])
                # Move files to new folder without overwriting
                if not os.path.exists(dest_file):
                    shutil.copy(source_path, dest_file)
        
                
    # write new JSON settings file
    if not os.path.exists(final_dir_json): os.makedirs(final_dir_json)
    json_file_name = os.path.split(file)[1]
    json_file_path = os.path.join(final_dir_json, json_file_name)
    with open(json_file_path, 'w', encoding='utf-8') as f:
        json.dump(new_json, f, ensure_ascii=False, indent=4)

#print(params['raw_json'])
#print(origin_paths)
#print(dest_paths)
#print(new_json['conditions'])
    
print('Job done')<jupyter_output><empty_output>