Skip to content

Commit bc4b63c

Browse files
author
mle
committed
initial commit, added scripts
1 parent f590dff commit bc4b63c

28 files changed

+1379
-0
lines changed

‎compare_2directories_linux.py

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import os
2+
import filecmp
3+
import sys
4+
import time
5+
6+
cwd = os.getcwd()
7+
list_files = os.listdir(cwd)
8+
9+
old_stdout = sys.stdout # save original stdout
10+
11+
dir1 = "/path/to/dir1"
12+
dir2 = "/path/to/dir2"
13+
14+
comparison = filecmp.dircmp(dir1, dir2)
15+
16+
# store differences in a file, in a python format
17+
f = open('report_differences.txt', 'w')
18+
sys.stdout = f # changing system output to the file
19+
comparison.report_full_closure()
20+
f.close()
21+
22+
sys.stdout = old_stdout # back to original system output
23+
24+
# read the differences to process them later
25+
with open('report_differences.txt', 'r') as f:
26+
rep = f.readlines()
27+
28+
star = "********************************************************************************\n"
29+
summa = []
30+
for ll in range(len(rep)):
31+
# get lines starting by "Only in..."
32+
if "Only in " in rep[ll]:
33+
summa.append(rep[ll])
34+
summa.append('\n')
35+
36+
if "Differing files :" in rep[ll]:
37+
# get lines starting by "diff" and "Differing"
38+
ii = ll
39+
while "diff " not in rep[ii]:
40+
ii -= 1
41+
summa.append(rep[ii])
42+
summa.append(rep[ll])
43+
summa.append('\n')
44+
45+
# process and write only interesting lines in the file
46+
with open('report_differences.txt', 'w') as f:
47+
f.write(str(time.ctime()) + "\n") # date time
48+
f.write(dir1 + "\n")
49+
f.write('vs.\n')
50+
f.write(dir2 + "\n\n\n")
51+
52+
for i in range(len(summa)):
53+
if "Only" in summa[i]:
54+
a, b = summa[i].split(":")
55+
f.write(star)
56+
f.write(a + ":\n") # a : "Only in ..."
57+
f.write(star)
58+
for l in b.split("'"): # b : "['file1,'file2'..]"
59+
if (("[" not in l) and ("]" not in l) and
60+
("," not in l)):
61+
# write only file names
62+
f.write(str(l) + "\n")
63+
f.write('\n')
64+
65+
if "Differing files :" in summa[i]:
66+
f.write(star)
67+
phra = summa[i - 1].split() # 'diff' 'path1' 'path2'
68+
a, b = summa[i].split(":")
69+
f.write(summa[i - 1]) # diff...
70+
listfiles = ""
71+
for l in b.split("'"):
72+
if (("[" not in l) and ("]" not in l) and
73+
("," not in l)):
74+
# listfiles += l+", "
75+
listfiles += l + "\n"
76+
# f.write("Differing files :"+listfiles[:-2]+"\n") # [-2] to remove ", "
77+
# print ("listfiles = "+str(listfiles)+"End")
78+
79+
f.write(star + "Differing files :\n" + star + listfiles[:-1] + "\n" + star) # [-2] to remove ", "
80+
81+
for l in b.split("'"):
82+
if (("[" not in l) and ("]" not in l) and
83+
("," not in l)):
84+
# write shell command to copy
85+
# toWrite=(l+"\n"+"tkdiff "+phra[1]+ "/"+l+
86+
toWrite = ("tkdiff " + phra[1] + "/" + l +
87+
" " + phra[2] + "/" + l + " &" + "\n")
88+
f.write(toWrite)
89+
f.write('\n')

‎compare_2directories_windows.py

+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
#! /usr/bin/python
2+
# -*- coding: utf-8 -*-
3+
4+
import filecmp
5+
import os
6+
import sys
7+
import time
8+
9+
cwd = os.getcwd()
10+
print(cwd)
11+
list_files = os.listdir(cwd)
12+
13+
old_stdout = sys.stdout # save original stdout
14+
15+
# --------------------------- User input
16+
# dir1 = os.path.abspath(os.path.realpath(os.path.join('C:','Users',...)))
17+
dir1 = "C:\\Users\\mat\\Documents\\00001"
18+
dir2 = "C:\\Users\\mat\\Documents\\00007"
19+
20+
directories_to_ignore = ['.svn', '.git', '.idea', 'docs']
21+
files_to_ignore = ['.pyc', '.log']
22+
write_command_for_compare = True
23+
24+
meld = "C:\\\\Program\ Files\ \(x86\)\\\\Meld\\\\Meld.exe " # tool to compare files
25+
26+
star = "********************************************************************************\n"
27+
# --------------------------- End User input
28+
29+
# --------------------------- Comparison, raw report
30+
comparison = filecmp.dircmp(dir1, dir2)
31+
32+
# store differences in a file, in a python format
33+
f = open('report_differences.txt', 'w')
34+
sys.stdout = f # changing system output to the file
35+
comparison.report_full_closure()
36+
f.close()
37+
38+
sys.stdout = old_stdout # back to original system output
39+
40+
# read the differences to process them later
41+
with open('report_differences.txt', 'r') as f:
42+
rep = f.readlines()
43+
# --------------------------- End of comparison, raw report
44+
for i in rep:
45+
print(i)
46+
# --------------------------- Process differences
47+
summa = []
48+
to_pass = False
49+
for ll in range(len(rep)):
50+
# get lines starting by "Only in..."
51+
if "Only in " in rep[ll]:
52+
# flag for ignoring directories
53+
for directo in directories_to_ignore:
54+
if directo in rep[ll]:
55+
to_pass = True
56+
break
57+
if to_pass:
58+
to_pass = False
59+
continue
60+
summa.append(rep[ll])
61+
summa.append('\n')
62+
63+
if "Differing files :" in rep[ll]:
64+
# get lines starting by "diff" and "Differing"
65+
ii = ll
66+
while "diff " not in rep[ii]:
67+
ii -= 1
68+
# flag for ignoring directories
69+
for directo in directories_to_ignore:
70+
if directo in rep[ii]:
71+
to_pass = True
72+
break
73+
if to_pass:
74+
to_pass = False
75+
continue
76+
summa.append(rep[ii])
77+
summa.append(rep[ll])
78+
summa.append('\n')
79+
80+
# process and write only interesting lines in the file
81+
cdirs = 0
82+
cfiles = 0
83+
with open('report_differences.txt', 'w') as f:
84+
f.write(str(time.ctime()) + "\n") # date time
85+
f.write(dir1 + "\n")
86+
f.write('vs.\n')
87+
f.write(dir2 + "\n\n\n")
88+
89+
for i in range(len(summa)):
90+
if "Only" in summa[i]:
91+
_, a, b = summa[i].split(":") # a:adress, b:list of files
92+
f.write(star)
93+
f.write("Only in C:" + a.replace("\\", "\\\\") + ":\n") # a : "Only in ..."
94+
cdirs += 1
95+
f.write(star)
96+
for l in b.split("'"): # b : "['file1,'file2'..]"
97+
if ("[" not in l) and ("]" not in l) and ("," not in l):
98+
# write only file names
99+
# flag for ignoring files
100+
for fti in files_to_ignore:
101+
if fti in l:
102+
to_pass = True
103+
break
104+
if to_pass:
105+
to_pass = False
106+
continue
107+
f.write(l + "\n") # str(l)
108+
cfiles += 1
109+
f.write('\n')
110+
111+
if "Differing files :" in summa[i]:
112+
# f.write(star)
113+
phra = summa[i - 1].split() # 'diff' 'path1' 'path2'
114+
a, b = summa[i].split(":") # a: adress, b:list of files
115+
# f.write(summa[i-1]) # diff...
116+
listfiles = ""
117+
for l in b.split("'"):
118+
if ("[" not in l) and ("]" not in l) and ("," not in l):
119+
# flag for files to ignore
120+
for fti in files_to_ignore:
121+
if fti in l:
122+
to_pass = True
123+
break
124+
if to_pass:
125+
to_pass = False
126+
continue
127+
listfiles += l + "\n"
128+
cfiles += 1
129+
130+
f.write(star + "Differing files between " + phra[1] + " and "
131+
+ phra[2] + "\n" + star + listfiles[:-1] + "\n" + star) # [-2] to remove ", "
132+
cdirs += 1
133+
134+
if write_command_for_compare:
135+
for l in b.split("'"):
136+
if ("[" not in l) and ("]" not in l) and ("," not in l):
137+
# flag for files to ignore
138+
for fti in files_to_ignore:
139+
if fti in l:
140+
to_pass = True
141+
break
142+
if to_pass:
143+
to_pass = False
144+
continue
145+
toWrite = ("nohup " + meld + # all the \\\ for windobws
146+
phra[1].replace("\\", "\\\\") + "\\\\" + l + " " +
147+
phra[2].replace("\\", "\\\\") + "\\\\" + l + " &" + "\n")
148+
f.write(toWrite)
149+
f.write('\n')
150+
151+
summary = 'Total of ' + str(cfiles) + ' differences in ' + str(cdirs) + ' directories.'
152+
f.write(summary)
153+
154+
# --------------------------- End of Process differences
155+
156+
# --------------------------- Process identical
157+
cdirs = 0
158+
cfiles = 0
159+
summa = []
160+
to_pass = False
161+
for ll in range(len(rep)):
162+
# get lines starting by "Identical files :"
163+
if "Identical files :" in rep[ll]:
164+
# flag for ignoring directories
165+
ii = ll
166+
while "diff " not in rep[ii]:
167+
ii -= 1
168+
# flag for ignoring directories
169+
for directo in directories_to_ignore:
170+
if directo in rep[ii]:
171+
to_pass = True
172+
break
173+
if to_pass:
174+
to_pass = False
175+
continue
176+
summa.append(rep[ii])
177+
summa.append(rep[ll])
178+
summa.append('\n')
179+
180+
# process and write only interesting lines in the file
181+
with open('report_identical.txt', 'w') as f:
182+
f.write(str(time.ctime()) + "\n") # date time
183+
f.write(dir1 + "\n")
184+
f.write('vs.\n')
185+
f.write(dir2 + "\n\n\n")
186+
187+
for i in range(len(summa)):
188+
if "Identical files :" in summa[i]:
189+
# f.write(star)
190+
phra = summa[i - 1].split() # 'diff' 'path1' 'path2'
191+
a, b = summa[i].split(":") # a: adress, b:list of files
192+
# f.write(summa[i-1]) # diff...
193+
listfiles = ""
194+
for l in b.split("'"):
195+
if ("[" not in l) and ("]" not in l) and ("," not in l):
196+
# flag for files to ignore
197+
for fti in files_to_ignore:
198+
if fti in l:
199+
to_pass = True
200+
break
201+
if to_pass:
202+
to_pass = False
203+
continue
204+
listfiles += l + "\n"
205+
cfiles += 1
206+
207+
f.write(star + "Identical files between " + phra[1] + " and "
208+
+ phra[2] + "\n" + star + listfiles[:-1] + "\n\n") # [-2] to remove ", "
209+
cdirs += 1
210+
summary = 'Total of ' + str(cfiles) + ' identical files in ' + str(cdirs) + ' directories.'
211+
f.write(summary)

‎compare_data_storage_solution.py

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
Created on Sat Mar 22 16:09:44 2014
4+
5+
@author: mat
6+
compare storage in binary files by size
7+
"""
8+
9+
import os
10+
import numpy as np
11+
import pickle
12+
13+
cwd = os.getcwd()
14+
15+
file01 = 'CV.csv' # 1307ko - example of file that does not exist anymore
16+
buffer_variable = np.loadtxt(file01, skiprows=2, delimiter=';') # np.table
17+
18+
19+
fich = open('pick', 'w')
20+
pickle.dump(buffer_variable, fich) # 3753ko
21+
fich.close()
22+
23+
np.savetxt('savetext.out', buffer_variable) # 4099ko
24+
25+
np.save('binarSave', buffer_variable) # 1306ko
26+
# recommended way!! loaded with np.load('file'+'npy')
27+
28+
buffer_variable.dump('binarDump') # 1306ko
29+
30+
buffer_variable.tofile('binarToFile') # 1306ko
31+
# loaded with: np.fromfile('file')

0 commit comments

Comments
 (0)