<jupyter_start><jupyter_text>Synthetic Data<jupyter_code>import csv
synthetic_data = tl.tensor(np.zeros([1000, 10, 20, 30], dtype='f'))

for i in range(200):
    start = time.time()
    with open('../Data/synthetic_data/data_normal/data{}.tensor'.format(i)) as file:
        reader = csv.reader(file, delimiter='\t')    
        for row in reader:
            indices = [[index] for index in np.int64(np.asarray(row[:-1]))-1]
            synthetic_data[tuple(indices)] = np.double(row[-1])
    print('>> synthetic_data{} loaded '.format(i), time.time() - start)<jupyter_output>>> synthetic_data0 loaded  0.8036372661590576
>> synthetic_data1 loaded  0.7937493324279785
>> synthetic_data2 loaded  0.791825532913208
>> synthetic_data3 loaded  0.7583863735198975
>> synthetic_data4 loaded  0.8618288040161133
>> synthetic_data5 loaded  0.7982239723205566
>> synthetic_data6 loaded  0.8991434574127197
>> synthetic_data7 loaded  1.0349154472351074
>> synthetic_data8 loaded  0.8069455623626709
>> synthetic_data9 loaded  0.8092312812805176
>> synthetic_data10 loaded  0.9315512180328369
>> synthetic_data11 loaded  0.8319196701049805
>> synthetic_data12 loaded  0.7559518814086914
>> synthetic_data13 loaded  0.7368896007537842
>> synthetic_data14 loaded  1.001147747039795
>> synthetic_data15 loaded  1.3474411964416504
>> synthetic_data16 loaded  0.8665621280670166
>> synthetic_data17 loaded  0.7760329246520996
>> synthetic_data18 loaded  0.7544364929199219
>> synthetic_data19 loaded  0.7518587112426758
>> synthetic_data20 loaded  0.8361003398895264
>> synthetic_data21 loade[...]<jupyter_text>Sample Video<jupyter_code>import csv
sample_video = tl.tensor(np.zeros([205, 240, 320, 3], dtype='d'))

for i in range(41):
    start = time.time()
    with open('../Data/sample_video/data/video{}.tensor'.format(i)) as file:
        reader = csv.reader(file, delimiter='\t')    
        for row in reader:
            indices = [[index] for index in np.int64(np.asarray(row[:-1]))-1]
            sample_video[tuple(indices)] = np.double(row[-1])
    print('>> sample_video{} loaded '.format(i), time.time() - start)<jupyter_output>>> sample_video0 loaded  17.943934202194214
>> sample_video1 loaded  18.175222873687744
>> sample_video2 loaded  18.29034399986267
>> sample_video3 loaded  18.185331106185913
>> sample_video4 loaded  18.23938298225403
>> sample_video5 loaded  18.205886602401733
>> sample_video6 loaded  18.270495891571045
>> sample_video7 loaded  18.050713300704956
>> sample_video8 loaded  18.23133420944214
>> sample_video9 loaded  18.222721338272095
>> sample_video10 loaded  18.293696880340576
>> sample_video11 loaded  18.25161385536194
>> sample_video12 loaded  18.053239583969116
>> sample_video13 loaded  18.167654991149902
>> sample_video14 loaded  18.11659526824951
>> sample_video15 loaded  18.15689444541931
>> sample_video16 loaded  18.233609199523926
>> sample_video17 loaded  18.1990008354187
>> sample_video18 loaded  18.265703439712524
>> sample_video19 loaded  18.208263397216797
>> sample_video20 loaded  17.94605541229248
>> sample_video21 loaded  18.154611587524414
>> sample_video22 loaded  18.[...]<jupyter_text>Stock<jupyter_code>import csv

stock2_tensor = tl.tensor(np.zeros([3089, 140, 5], dtype=float))
start = time.time()
with open('../Data/stock/KOSPI140/Stock.tensor') as file:
    reader = csv.reader(file, delimiter='\t')    
    for row in reader:
        indices = np.asarray([index for index in np.int64(np.asarray(row[:-1]))])[[1, 0, 2]]
        stock2_tensor[tuple(indices)] = np.double(row[-1])
print('>> stock tensor loaded ', time.time() - start)<jupyter_output>>> stock tensor loaded  36.043638944625854<jupyter_text>Hall from OLSTEC* `hall1-200.mat`: 144 * 176 * 200 => 200 * 144 * 176* `hall_144x100_frame2900-3899_pan.mat`: 144 * 100 * 1000 => 1000 * 144 * 100<jupyter_code>from scipy.io import loadmat
import h5py

hall1 = loadmat('../Data/hall/hall1-200.mat')['XO']
hall1 = np.moveaxis(hall1, -1, 0)
hall1 = hall1.reshape(200, 144, 176, order='F')

with h5py.File('../Data/hall/hall_144x100_frame2900-3899_pan.mat', 'r') as f:
    hall2 = np.array(f['X0'])
    hall2 = hall2.reshape(1000, 144, 100, order='F')
    
hall1_tensor = tl.tensor(hall1, dtype='f')
hall2_tensor = tl.tensor(hall2, dtype='f')
print(hall1_tensor.shape, hall2_tensor.shape)
# make_video(hall1, 'hall1.avi', False)
# make_video(hall2, 'hall2.avi', False)<jupyter_output>(200, 144, 176) (1000, 144, 100)<jupyter_text>Air Quality Tensor* `   (measurement)`* Beijing Air Quality  * 2,454,305 out of 2,524,536 (35,063 * 12 * 6)* Korea Air Quality  * 11,270,028 out of 18,368,364 (9,478 * 323 * 6)* Madrid Air Quality  * 8,036,759 out of 21,587,328 (64,248 * 24 * 14)<jupyter_code># beijing_df = pd.read_csv('../../Data/air_quality/BeijingAirQuality/beijing.tensor', delimiter='\t', header=None)
korea_df = pd.read_csv('../../Data/air_quality/KoreaAirQuality/korea_airquality.tensor', delimiter='\t', header=None)
# madrid_df = pd.read_csv('../../Data/air_quality/MadridAirQuality/1hour_madrid.tensor', delimiter='\t', header=None)
def get_tensor(df):
    start = time.time()

    dims = df[[0,1,2]].max()+1
    tensor = np.empty(dims) * np.nan
    tensor.shape

    for i, row in df.iterrows():
        indices = [[index] for index in np.int64(np.asarray(row[:-1]))]
        tensor[tuple(indices)] = np.double(row[3])

    avg = []
    for i in range(tensor.shape[2]):
        avg.append(np.nanmean(tensor[:,:,i]))
    inds = np.where(np.isnan(tensor))
    for ind in zip(inds[0], inds[1], inds[2]):
        tensor[ind] = avg[ind[-1]]
        
    print(time.time() - start)
    return tensor
# beijing_tensor = get_tensor(beijing_df)
korea_tensor = get_tensor(korea_df)
# madrid_tensor = get_tensor(madrid_df)
with open('results_0128.p', 'wb') as fp:
    pickle.dump(results, fp, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>--- Experiment 1<jupyter_code>from online_tensor_decomposition_fcp_abridged06 import *
# results = {}
# synthetic
(n_iter, ul, ll) = (1, 1.2, 1.1)
tensor_stream = create_tensor_stream(synthetic_data, start_to_stream=10, batch_sizes=np.full((99), 10, dtype=int))
# results['synthetic-10'] = online_tensor_decomposition('synthetic', synthetic_data, tensor_stream, rank=10, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['synthetic-20'] = online_tensor_decomposition('synthetic', synthetic_data, tensor_stream, rank=20, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['synthetic-30'] = online_tensor_decomposition('synthetic', synthetic_data, tensor_stream, rank=30, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['synthetic-40'] = online_tensor_decomposition('synthetic', synthetic_data, tensor_stream, rank=40, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['synthetic-50'] = online_tensor_decomposition('synthetic', synthetic_data, tensor_stream, rank=50, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
datasets = ('synthetic-20', 'synthetic-30', 'synthetic-40')
plot_global(datasets, 'E1_synthetic', 35)
plot_local(datasets, 'E1_synthetic', 35)
# video
(n_iter, ul, ll) = (1, 6, 2)
tensor_stream = create_tensor_stream(sample_video, start_to_stream=5, batch_sizes=np.full((40), 5, dtype=int))
# results['video-10'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=10, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['video-20'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=20, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['video-30'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=30, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['video-40'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=40, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['video-50'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=50, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
datasets = ('video-20', 'video-30', 'video-40')
plot_global(datasets, 'E1_video', 35)
plot_local(datasets, 'E1_video', 35)
# stock
(n_iter, ul, ll) = (1, 5, 3)
tensor_stream = create_tensor_stream(stock2_tensor, start_to_stream=5, batch_sizes=np.full((1028), 3, dtype=int))
results['stock-25'] = online_tensor_decomposition('stock', stock2_tensor, tensor_stream, rank=25, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['stock-30'] = online_tensor_decomposition('stock', stock2_tensor, tensor_stream, rank=30, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['stock-35'] = online_tensor_decomposition('stock', stock2_tensor, tensor_stream, rank=35, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
datasets = ('stock-20', 'stock-22', 'stock-24')
plot_global(datasets, 'E1_stock', 50)
plot_local(datasets, 'E1_stock', 45)
# hall
(n_iter, ul, ll) = (1, 0.5, 0.1)
tensor_stream = create_tensor_stream(hall1_tensor, start_to_stream=10, batch_sizes=np.full((19), 10, dtype=int))
# results['hall-10'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=15, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['hall-15'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=20, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['hall-20'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=20, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['hall-25'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=25, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['hall-30'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=30, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['hall-35'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=35, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['hall-40'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=40, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
#results['hall-45'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=45, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
datasets = ('hall-30', 'hall-35', 'hall-40')
plot_global(datasets, 'E1_hall', 40)
plot_local(datasets, 'E1_hall', 40)
# korea air-quality
(n_iter, ul, ll) = (1, 2, 1.3)
tensor_stream = create_tensor_stream(korea_tensor, start_to_stream=79, batch_sizes=np.full((94), 100, dtype=int))
# results['korea-10'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=10, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['korea-15'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=15, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['korea-20'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
# results['korea-25'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=25, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['korea-30'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=30, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['korea-40'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=40, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
results['korea-50'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=50, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
datasets = ('korea-30', 'korea-40', 'korea-50')
plot_global(datasets, 'E1_korea', 35)
plot_local(datasets, 'E1_korea', 35)<jupyter_output><empty_output><jupyter_text>--- Experiment 2<jupyter_code>from matplotlib import colors
def make_rgb_transparent(color, alpha=0.6, bg_rgb=(1,1,1)):
    rgb = colors.colorConverter.to_rgb(color)
    return [alpha * c1 + (1 - alpha) * c2
            for (c1, c2) in zip(rgb, bg_rgb)]

def plot_mem(datasets, name):
    colors = ('dodgerblue','mediumseagreen', 'hotpink', '#fba84a')
    libs = ("dao", "dtd", "ocp", 'fcp')
    patterns = ( "" , "\\\\\\\\\\" , "////" , "xxxx")
    markers = ("o", "x", "s", "^", "4")
    
    index = np.arange(5)
    bar_width = 0.2

    
    # create plot
    fig, ax1 = plt.subplots(figsize = (6, 4), dpi = 150)
    plt.xticks(index + bar_width*1.5, ('Synthetic', 'Video', 'Stock', 'Hall', 'Korea'))
    plt.rcParams['hatch.linewidth'] = 0.2

    for i, (color, lib) in enumerate(zip(colors, libs)):
        mem_list = [results[dataset][lib][4] for dataset in datasets]
        
        rects1 = ax1.bar(index + bar_width*i, mem_list, bar_width, color=make_rgb_transparent(color, alpha=0.0), label=lib, edgecolor='black', hatch=patterns[i], linewidth=0.5)

    ax1.set_xlabel('Datasets')
    ax1.set_ylabel('Memory Usage (byte)')
    ax1.set_yscale('log')

    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

    for i, (color, lib) in enumerate(zip(colors, libs)):
        acc_list = [results[dataset][lib][0] for dataset in datasets]
        for j, acc in enumerate(acc_list):
            if j == 4:
                ax2.scatter(index[j] + bar_width*i, acc, 70, color=colors[i], marker=markers[j], linewidth=2)
            elif j == 1: 
                ax2.scatter(index[j] + bar_width*i, acc, 50, color=colors[i], marker=markers[j], linewidth=2)
            else:
                ax2.scatter(index[j] + bar_width*i, acc, 50, color=colors[i], marker=markers[j], facecolors='none', linewidth=2)
    ax2.tick_params(axis='y')

    ax2.set_ylabel('Global Fitness', rotation=270, labelpad=15)

    fig.tight_layout()  # otherwise the right y-label is slightly clipped
    plt.show()
#     plt.savefig(f'./plots/{name}_mem.pdf', bbox_inches = 'tight', pad_inches = 0)
plot_mem(('synthetic-30', 'video-30', 'stock-20', 'hall-30', 'korea-40'), 'E2')
from matplotlib import colors
def make_rgb_transparent(color, alpha=0.6, bg_rgb=(1,1,1)):
    rgb = colors.colorConverter.to_rgb(color)
    return [alpha * c1 + (1 - alpha) * c2
            for (c1, c2) in zip(rgb, bg_rgb)]

def plot_mem(datasets, name):
    colors = ('dodgerblue','mediumseagreen', 'hotpink', '#fba84a')
    libs = ("dao", "dtd", "ocp", 'fcp')
    patterns = ( "" , "\\\\\\\\\\" , "////" , "xxxx")
    markers = ("o", "x", "s", "^", "4")
    
    index = np.arange(5)
    bar_width = 0.2

    
    # create plot
    fig, ax1 = plt.subplots(figsize = (6, 4), dpi = 150)
    plt.xticks(index + bar_width*1.5, ('Synthetic', 'Video', 'Stock', 'Hall', 'Korea'))
    plt.rcParams['hatch.linewidth'] = 0.2

    for i, (color, lib) in enumerate(zip(colors, libs)):
        mem_list = [results[dataset][lib][4] for dataset in datasets]
        print(mem_list)
        
        rects1 = ax1.bar(index + bar_width*i, mem_list, bar_width, color=make_rgb_transparent(color, alpha=0.7), label=lib, edgecolor='black', hatch=patterns[i], linewidth=0.5)

    ax1.set_xlabel('Datasets')
    ax1.set_ylabel('Memory Usage (byte)')
    ax1.set_yscale('log')

    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

    for i, dataset in enumerate(datasets):
        acc_list = [results[dataset][lib][0] for lib in libs]   
        ax2.plot(i + bar_width*index[:4], acc_list, marker="o", color='black', zorder=1)
    for i, (color, lib) in enumerate(zip(colors, libs)):
        acc_list = [results[dataset][lib][0] for dataset in datasets]        
        ax2.scatter(index + bar_width*i, acc_list, 30, color='black', marker="o", facecolor=colors[i], linewidth=1.3, zorder=2)
    ax2.tick_params(axis='y')

    ax2.set_ylabel('Global Fitness', rotation=270, labelpad=13)

    fig.tight_layout()  # otherwise the right y-label is slightly clipped
#     plt.show()
    plt.savefig(f'./plots/{name}_mem.svg', bbox_inches = 'tight', pad_inches = 0)
    
    print([results[dataset]['fcp'][4]/results[dataset]['dao'][4] for dataset in datasets])
plot_mem(('synthetic-30', 'video-30', 'stock-20', 'hall-30', 'korea-40'), 'E2')<jupyter_output>[199424, 744488, 557472, 469496, 2024776]
[270160, 320800, 541736, 202696, 3244936]
[2717056, 94372576, 114616, 892536, 10744216]
[24000232, 377856232, 17298616, 20275416, 146962632]
[120.34776155327343, 507.53837805310496, 31.030466104127203, 43.18549252815785, 72.5821681015579]<jupyter_text>--- Experiment 5<jupyter_code>from online_tensor_decomposition_fcp01 import *
results = {}

(n_iter, ul, ll) = (1, 6, 2)
tensor_stream = create_tensor_stream(sample_video, start_to_stream=5, batch_sizes=np.full((40), 5, dtype=int))
results['video'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=30, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
def plot_E5_error(dataset):
    
    markers = ("+", "x", "1", "2")
    markers = ("o", "x", "s", "^")
    colors = ('dodgerblue','mediumseagreen', 'hotpink', '#fba84a')
    libs = ("dao", "dtd", "ocp", "fcp")
    fig = plt.figure(figsize = (7, 3), dpi = 150,) 
    plt.ylabel('Local Error Norm', fontsize=12)
    plt.xlabel('# of Stacked Slices', fontsize=12)
    #         ax1.xaxis.set_label_position('top')
    
    
    split_points, refine_points = results[dataset]['dao'][6]
    for p in refine_points:
        plt.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='--')
    for p in split_points:
        plt.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='-')
    
    for color, marker, lib in zip(colors, markers, libs):
        verbose_list = results[dataset][lib][5]
        plt.plot(verbose_list[:,0], verbose_list[:,2], linewidth=1, marker=marker, color=color, markersize=4)
    plt.savefig('plots/E5_{}_error.pdf'.format(dataset), bbox_inches='tight', pad_inches=0)
    
plot_E5_error('video')
    
def plot_E5_rt(dataset):
    markers = ("+", "x", "1", "2")
    markers = ("o", "x", "s", "^")
    colors = ('dodgerblue','mediumseagreen', 'hotpink', '#fba84a')
    libs = ("dao", "dtd", "ocp", "fcp")
    plt.figure(figsize = (7, 3), dpi = 150,) 
    
    plt.yscale('log')
    plt.ylabel('Local Running Time (s)', fontsize=12)
    plt.xlabel('# of Stacked Slices', fontsize=12)
    #         ax1.xaxis.set_label_position('top')
    
    
    split_points, refine_points = results[dataset]['dao'][6]
    for p in refine_points:
        plt.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='--')
    for p in split_points:
        plt.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='-')
    
    for color, marker, lib in zip(colors, markers, libs):
        verbose_list = results[dataset][lib][5]
        plt.plot(verbose_list[:,0], verbose_list[:,1], linewidth=1, marker=marker, color=color, markersize=4)
        plt.savefig('plots/E5_{}_rt.pdf'.format(dataset), bbox_inches='tight', pad_inches=0)
        
plot_E5_rt('video')
from matplotlib import gridspec

def plot_E5(dataset):
    
    markers = ("x", "1", "2", "+")
    markers = ("x", "s", "^", "o")
    colors = ('mediumseagreen', 'hotpink', '#fba84a', 'dodgerblue')
    libs = ("dtd", "ocp", "fcp", "dao")
    
    fig = plt.figure(figsize = (9, 6), dpi = 150)
    
    gs = gridspec.GridSpec(2, 1, height_ratios=[1.5, 1]) 
    ax1 = plt.subplot(gs[0])
    
    ax1.set_ylabel('Local Error Norm', fontsize=12)
#     ax1.set_xlabel('# of Stacked Slices', fontsize=12)
    #         ax1.xaxis.set_label_position('top')
    
    
    split_points, refine_points = results[dataset]['dao'][6]
    for p in refine_points:
        ax1.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='--')
    for p in split_points:
        ax1.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='-')
    
    for color, marker, lib in zip(colors, markers, libs):
        verbose_list = results[dataset][lib][5]
        ax1.plot(verbose_list[:,0], verbose_list[:,2], linewidth=1, marker=marker, color=color, markersize=4)
        
    ax2 = plt.subplot(gs[1], sharex = ax1)
    ax2.set_yscale('log')
    ax2.set_ylabel('Local Running\nTime (s)', fontsize=12)
    ax2.set_xlabel('# of Stacked Slices', fontsize=12)
    #         ax1.xaxis.set_label_position('top')
    
    split_points, refine_points = results[dataset]['dao'][6]
    for p in refine_points:
        ax2.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='--')
    for p in split_points:
        ax2.axvline(p,  label='line: {}'.format(p), c='lightgray', linewidth=2, linestyle='-')
    
    for color, marker, lib in zip(colors, markers, libs):
        verbose_list = results[dataset][lib][5]
        ax2.plot(verbose_list[:,0], verbose_list[:,1], linewidth=1, marker=marker, color=color, markersize=4)
        
    plt.setp(ax1.get_xticklabels(), visible=False)
    plt.subplots_adjust(hspace=.0)
#     fig.tight_layout()  # otherwise the right y-label is slightly clipped
    plt.savefig('plots/E5_{}.svg'.format(dataset), bbox_inches='tight', pad_inches=0)
    plt.show()
    
plot_E5('video')<jupyter_output><empty_output><jupyter_text>--- Experiment 3<jupyter_code>tensor_stream = create_tensor_stream(korea_tensor, start_to_stream=79, batch_sizes=np.full((94), 100, dtype=int))
results['korea--'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=-1, ll=-1, verbose=False)
# korea air-quality
tensor_stream = create_tensor_stream(korea_tensor, start_to_stream=79, batch_sizes=np.full((94), 100, dtype=int))
results['korea-split-1.0'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=1.0, ll=-1, verbose=False) # 65
results['korea-split-1.2'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=1.2, ll=-1, verbose=False) # 57
results['korea-split-1.4'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=1.4, ll=-1, verbose=False) # 45
results['korea-split-1.6'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=1.6, ll=-1, verbose=False) # 36
results['korea-split-1.8'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=1.8, ll=-1, verbose=False) # 5
# korea air-quality
tensor_stream = create_tensor_stream(korea_tensor, start_to_stream=79, batch_sizes=np.full((94), 100, dtype=int))
results['korea-refine-1.4'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=-1, ll=1.4, verbose=False) # 15
results['korea-refine-1.6'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=-1, ll=1.6, verbose=False) # 14
results['korea-refine-1.8'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=-1, ll=1.8, verbose=False) # 12
results['korea-refine-2.0'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=-1, ll=2.0, verbose=False) # 8
results['korea-refine-2.2'] = online_tensor_decomposition('korea', korea_tensor, tensor_stream, rank=20, methods=['dao'], n_iter=n_iter, ul=-1, ll=2.2, verbose=False) # 6
for dataset in ('korea-split-1.0', 'korea-split-1.2', 'korea-split-1.4', 'korea-split-1.6', 'korea-split-1.8'):
    plot_E5(dataset)
for dataset in ('korea-refine-1.4', 'korea-refine-1.6', 'korea-refine-1.8', 'korea-refine-2.0', 'korea-refine-2.2'):
    plot_E5(dataset)
rows = []
for dataset in ('korea-split-1.0', 'korea-split-1.2', 'korea-split-1.4', 'korea-split-1.6', 'korea-split-1.8'):
    row = dataset.split('-')
    row.append('-')
    verbose_list = results[dataset]['dao'][5]
    split_points, refine_points = results[dataset]['dao'][6]
    row.append(len(split_points))
    row += results[dataset]['dao'][:5]
    rows.append(row)
    
for dataset in ('korea-refine-1.4', 'korea-refine-1.6', 'korea-refine-1.8', 'korea-refine-2.0', 'korea-refine-2.2'):
    row = dataset.split('-')[:-1]
    row.append('-')
    row.append(dataset.split('-')[-1])
    verbose_list = results[dataset]['dao'][5]
    split_points, refine_points = results[dataset]['dao'][6]
    row.append(len(refine_points))
    row += results[dataset]['dao'][:5]
    rows.append(row)
    
dataset = 'korea--'
row = dataset.split('-')[:1]
row += ['-'] * 3
verbose_list = results[dataset]['dao'][5]
split_points, refine_points = results[dataset]['dao'][6]
row.append(len(refine_points))
row += results[dataset]['dao'][:5]
rows.append(row)
    
df = pd.DataFrame(rows, columns=['dataset', 'process', 'ul', 'll', 'points-#', 'global-fit', 'local-fit', 'global-rt', 'local-rt', 'mem'])
df<jupyter_output><empty_output><jupyter_text>--- Experiment 4<jupyter_code># hall
(n_iter, ul, ll) = (1, 0.5, 0.1)
tensor_stream = create_tensor_stream(hall1_tensor, start_to_stream=10, batch_sizes=np.full((19), 10, dtype=int))
results['hall'] = online_tensor_decomposition('hall', hall1_tensor, tensor_stream, rank=100, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
frame = results['hall']['dao'][6][0][10]*10
for lib in ("dao", "dtd", "ocp", 'fcp'):
    make_video(results['hall'][lib][-1], f'plots/E4-hall-{lib}.avi', False)
    imwrite(f'plots/E4-hall-{lib}-{frame}.jpg', results['hall'][lib][-1][frame])
make_video(hall1_tensor, 'plots/E4-hall-org.avi', False)
imwrite(f'plots/E4-hall-org-{frame}.jpg', hall1_tensor[frame])
# video
(n_iter, ul, ll) = (1, 6, 2)
tensor_stream = create_tensor_stream(sample_video, start_to_stream=5, batch_sizes=np.full((40), 5, dtype=int))
results['video'] = online_tensor_decomposition('video', sample_video, tensor_stream, rank=100, n_iter=n_iter, ul=ul, ll=ll, verbose=False)
frame = results['video']['dao'][6][0][0]*5
for lib in ("dao", "dtd", "ocp", 'fcp'):
    make_video(results['video'][lib][-1], f'plots/E4-video-{lib}.avi')
    imwrite(f'plots/E4-video-{lib}-{frame}.jpg', results['video'][lib][-1][frame])
make_video(sample_video, 'plots/E4-video-org.avi')
imwrite(f'plots/E4-video-org-{frame}.jpg', sample_video[frame])
for frame in range(205):
    imwrite(f'plots/video/org-{frame}.jpg', sample_video[frame])<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup