<jupyter_start><jupyter_text>IID case: all the clients have images of all the classes  Centralized topology<jupyter_code># IID case: all the clients have images of all the classes
# Centralized graph topology (fully-connected network)

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('centralized', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs)
plt.show()
graph = nx.from_numpy_matrix(comm_matrix)
nx.draw(graph)<jupyter_output><empty_output><jupyter_text>Ring topology<jupyter_code># IID case: all the clients have images of all the classes
# Ring graph topology: each client is connected to two neighbours exactly

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('ring', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs1 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs1)
plt.show()
graph = nx.from_numpy_matrix(comm_matrix)
nx.draw(graph)<jupyter_output><empty_output><jupyter_text>Grid topology<jupyter_code># IID case: all the clients have images of all the classes
# Grid graph topology: each client is connected to exactly 4 neighbours

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('grid', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs2 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs2)
plt.show()
graph = nx.from_numpy_matrix(comm_matrix)
nx.draw(graph)<jupyter_output><empty_output><jupyter_text>Star topology<jupyter_code># IID case: all the clients have images of all the classes
# Star graph topology: each client is connected only to one central machine

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('star', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs3 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs3)
plt.show()
graph = nx.from_numpy_matrix(comm_matrix)
nx.draw(graph)<jupyter_output><empty_output><jupyter_text>Circular ladder topology<jupyter_code># IID case: all the clients have images of all the classes
# Circular ladder topology: each client is connected to exactly 3 neighbours

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('ladder', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs4 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs4)
plt.show()
graph = nx.from_numpy_matrix(comm_matrix)
nx.draw(graph)<jupyter_output><empty_output><jupyter_text>Disconnected graph<jupyter_code># IID case: all the clients have images of all the classes
# Disconnected graph: no communication between machines

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('disconnected', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs5 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs5)
plt.show()<jupyter_output>0-th round
lr 0.1 | average train loss 2.29 | test loss 2.29 | test acc: 0.276
1-th round
lr 0.1 | average train loss 2.24 | test loss 2.24 | test acc: 0.429
2-th round
lr 0.1 | average train loss 1.81 | test loss 1.67 | test acc: 0.713
3-th round
lr 0.1 | average train loss 1.12 | test loss 0.834 | test acc: 0.808
4-th round
lr 0.1 | average train loss 0.791 | test loss 0.584 | test acc: 0.850
5-th round
lr 0.1 | average train loss 0.608 | test loss 0.463 | test acc: 0.876
6-th round
lr 0.1 | average train loss 0.485 | test loss 0.39 | test acc: 0.895
7-th round
lr 0.1 | average train loss 0.359 | test loss 0.343 | test acc: 0.906
8-th round
lr 0.1 | average train loss 0.314 | test loss 0.311 | test acc: 0.912
9-th round
lr 0.1 | average train loss 0.197 | test loss 0.29 | test acc: 0.916
[0.381028, 0.639411, 0.398183, 0.419525, 0.429052, 0.546445, 0.40172, 0.418679, 0.394743, 0.396699, 0.59928, 0.410585, 0.442971, 0.596245, 0.427828, 0.462024, 0.442234, 0.412808, 0.413047, 0.424196, [...]<jupyter_text>Plot<jupyter_code>fig, ax = plt.subplots(1, figsize=(12, 9))
ax.set_ylim([0, 1])
x = np.array(range(10))
ax.plot(x, accs, color="blue", label="centralized")
ax.plot(x, accs1, color="green", label="ring")
ax.plot(x, accs2, color="purple", label="grid")
ax.plot(x, accs3, color="yellow", label="star")
ax.plot(x, accs4, color="orange", label="ladder")
ax.plot(x, accs5, color="red", label="disconnected")

# Add a legend, and position it on the lower right (with no box)
plt.legend(loc="lower right", title="Graph topology")
plt.title("Accuracy curve depending on the network topology, IID case")
plt.xlabel("Round")
plt.ylabel("Accuracy")
plt.show()<jupyter_output><empty_output><jupyter_text>NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].<jupyter_code># NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].
# Centralized graph topology (fully-connected network)

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('centralized', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients, 'non-iid')

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs)
plt.show()
# NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].
# Ring graph topology: each client is connected to two neighbours exactly

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('ring', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients, 'non-iid')

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs1 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs1)
plt.show()
# NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].
# Grid graph topology: each client is connected to exactly 4 neighbours

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('grid', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients, 'non-iid')

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs2 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs2)
plt.show()
# NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].
# Star graph topology: each client is connected only to one central machine

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('star', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients, 'non-iid')

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs3 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs3)
plt.show()
# NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].
# Circular ladder topology: each client is connected to exactly 3 neighbours

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('ladder', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients, 'non-iid')

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs4 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs4)
plt.show()
# NON-IID case: every client has images of two categories chosen from [0, 1], [2, 3], [4, 5], [6, 7], or [8, 9].
# Disconnected graph: no communication between machines

# Hyperparameters

num_clients = 100
num_rounds = 10
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('disconnected', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients, 'non-iid')

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs5 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs5)
plt.show()
fig, ax = plt.subplots(1, figsize=(12, 9))
ax.set_ylim([0, 1])
x = np.array(range(10))
ax.plot(x, accs, color="blue", label="centralized")
ax.plot(x, accs1, color="green", label="ring")
ax.plot(x, accs2, color="purple", label="grid")
ax.plot(x, accs3, color="yellow", label="star")
ax.plot(x, accs4, color="orange", label="ladder")
ax.plot(x, accs5, color="red", label="disconnected")

# Add a legend, and position it on the lower right (with no box)
plt.legend(loc="lower right", title="Graph topology")
plt.title("Accuracy curve depending on the network topology, non-IID case")
plt.xlabel("Round")
plt.ylabel("Accuracy")
plt.show()<jupyter_output><empty_output><jupyter_text>Topology evolving over time<jupyter_code># IID case: all the clients have images of all the classes
# Grid graph topology: each client is connected to exactly 4 neighbours
# Base case with a fixed topology for comparison

# Hyperparameters

num_clients = 100
num_rounds = 25
epochs = 1
batch_size = 32

# Communication matrix

comm_matrix = create_mixing_matrix('grid', num_clients)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs0 = run(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients)

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs0)
plt.show()
# IID case: all the clients have images of all the classes
# We assign a random order to the topologies and change the topology at each round according to this order

# Hyperparameters

num_clients = 100
num_rounds = 25
epochs = 1
batch_size = 32
topos = ['centralized', 'ring', 'star', 'grid', 'ladder']
np.random.shuffle(topos)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs = run_topos(train_loader, test_loader, num_rounds,
                                              epochs, num_clients, topos, shuffle='modulo')

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs)
plt.show()
# IID case: all the clients have images of all the classes
# We keep the same topology for a number of rounds before changing it

# Hyperparameters

num_clients = 100
num_rounds = 25
epochs = 1
batch_size = 32
topos = ['centralized', 'ring', 'star', 'grid', 'ladder']
np.random.shuffle(topos)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs1 = run_topos(train_loader, test_loader, num_rounds,
                                               epochs, num_clients, topos, shuffle='fraction')

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs1)
plt.show()
# IID case: all the clients have images of all the classes
# We pick a topology at random at each round

# Hyperparameters

num_clients = 100
num_rounds = 25
epochs = 1
batch_size = 32
topos = ['centralized', 'ring', 'star', 'grid', 'ladder']
np.random.shuffle(topos)

# Creating decentralized datasets

train_loader, test_loader = load_data(batch_size, num_clients)

# Instantiate models and optimizers and run decentralized training

global_model, client_models, accs2 = run_topos(train_loader, test_loader, num_rounds,
                                               epochs, num_clients, topos, shuffle='random')

cons = consensus(global_model, client_models)
print(cons)

axes = plt.gca()
axes.set_ylim([0,1])
plt.plot(range(num_rounds), accs2)
plt.show()
fig, ax = plt.subplots(1, figsize=(12, 9))
ax.set_ylim([0, 1])
x = np.array(range(25))
ax.plot(x, accs, color="blue", label="modulo")
ax.plot(x, accs1, color="green", label="fraction")
ax.plot(x, accs2, color="purple", label="random")
ax.plot(x, accs0, color="red", label="centralized (ref)")

# Add a legend, and position it on the lower right (with no box)
plt.legend(loc="lower right", title="Topology selection")
plt.title("Accuracy curve when the network topology changes over time, IID case")
plt.xlabel("Round")
plt.ylabel("Accuracy")
plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup