diff --git a/migration/download-files-from-url.py b/migration/download-files-from-url.py new file mode 100644 index 0000000000000000000000000000000000000000..65b8ecde886c9272587ef8b295fdcb84bf159901 --- /dev/null +++ b/migration/download-files-from-url.py @@ -0,0 +1,48 @@ +import urllib3 +import os +import sys +import re + +# python /pod/REPRISE/esup-pod-api-sample/migration/download-files-from-url.py + +# ADAPT - Videos base path +base_files_path="/pod/media/files" + +# ADAPT - Input file +media_files_list="/pod/REPRISE/esup-pod-api-sample/migration/media_files_list.txt" + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +http = urllib3.PoolManager(cert_reqs='CERT_NONE') +http_headers = urllib3.make_headers(basic_auth='julien.marchal@esup-portail.org:XXXXXX') + +print("***************** USE media_files_list : %s *************"%media_files_list) + +fmedia = open(media_files_list, 'r') +lines = fmedia.readlines() + +for l in lines: + l = l.strip() + parts = re.search('^https://(.*)/media/files/(.*)/(.*)$', l, re.IGNORECASE) + domain=parts.group(1) + hashkey=parts.group(2) + filename=parts.group(3) + #print("%s - %s" % (hashkey,filename )) + + hashPath=base_files_path+"/"+hashkey + outFilePath =hashPath+"/"+filename + + if not os.path.exists(hashPath): + os.mkdir(hashPath) + + print("download file : %s " % (l), end="", flush=True) + r = http.request('GET', l, headers=http_headers, preload_content=False) + with open(outFilePath, 'wb') as out: + while True: + data = r.read(8192) + if not data: + break + out.write(data) + #print(".", end="", flush=True) + r.release_conn() + print(" DONE") + diff --git a/migration/download-videos-from-url.py b/migration/download-videos-from-url.py new file mode 100644 index 0000000000000000000000000000000000000000..60fdb95eb406832c7e473fec54cf5864b06aa159 --- /dev/null +++ b/migration/download-videos-from-url.py @@ -0,0 +1,62 @@ +import urllib3 +import os +import sys +import re + +# python /pod/REPRISE/esup-pod-api-sample/migration/download-videos-from-url.py + +# ADAPT - Videos base path +base_videos_path="/pod/media/videos" + +# ADAPT - Input file +media_videos_list="/pod/REPRISE/esup-pod-api-sample/migration/media_videos_list.txt" + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +http = urllib3.PoolManager(cert_reqs='CERT_NONE') +http_headers = urllib3.make_headers(basic_auth='julien.marchal@esup-portail.org:XXXXX') + +print("***************** USE videos_files_list : %s *************"%media_videos_list) + +fvideo = open(media_videos_list, 'r') +lines = fvideo.readlines() + +for l in lines: + l = l.strip() + if re.match(r'^https://(.*)/media/videos/(.*)/(.*)/(.*)$', l): + parts = re.search('^https://(.*)/media/videos/(.*)/(.*)/(.*)$', l, re.IGNORECASE) + domain=parts.group(1) + hashkey=parts.group(2) + dirname=parts.group(3) + filename=parts.group(4) + #print("%s - %s - %s" % (hashkey,dirname,filename )) + hashPath = base_videos_path+"/"+hashkey + videoIdPath = hashPath+"/"+dirname + outFilePath = videoIdPath+"/"+filename + + if not os.path.exists(hashPath): + os.mkdir(hashPath) + if not os.path.exists(videoIdPath): + os.mkdir(videoIdPath) + + elif re.match(r'^https://(.*)/media/videos/(.*)/(.*)$', l): + parts = re.search('^https://(.*)/media/videos/(.*)/(.*)$', l, re.IGNORECASE) + domain=parts.group(1) + hashkey=parts.group(2) + filename=parts.group(3) + #print("%s - %s" % (hashkey,filename )) + hashPath = base_videos_path+"/"+hashkey + outFilePath = hashPath+"/"+filename + if not os.path.exists(hashPath): + os.mkdir(hashPath) + + print("download video : %s " % (l), end="", flush=True) + r = http.request('GET', l, headers=http_headers, preload_content=False) + with open(outFilePath, 'wb') as out: + while True: + data = r.read(16384) + if not data: + break + out.write(data) + #print(".", end="", flush=True) + r.release_conn() + print(" DONE") \ No newline at end of file diff --git a/migration/list-files-url.py b/migration/list-files-url.py new file mode 100644 index 0000000000000000000000000000000000000000..b7dd7810e2fb02e1ac20fae55401eef400babd1a --- /dev/null +++ b/migration/list-files-url.py @@ -0,0 +1,29 @@ +import os +from django.contrib.auth.models import User +from django.contrib.sites.models import Site + +# cd /pod/src # GOTO pod src directoy (where is manage.py) +# python manage.py shell < /pod/REPRISE/esup-pod-api-sample/migration/list-files-url.py > /pod/REPRISE/esup-pod-api-sample/migration/media_files_list.txt + +# ADAPT - Domain of users and videos +site_domain_keep="mediatheque-pedagogique-test.unicaen.fr" + +# ADAPT - Videos base path +base_files_path="/pod/media/files" + +keep_site= Site.objects.filter(domain=site_domain_keep) + +files_list=[] + +users=User.objects.all() +for user in users: + username=user.username + userkey=user.owner.hashkey + if(user.owner.sites.filter(domain=site_domain_keep).exists()) : + user_files_path=base_files_path+"/"+userkey + if os.path.isdir(user_files_path): + for f in os.listdir(user_files_path): + file_url="https://%s/media/files/%s/%s" % (site_domain_keep,userkey,f) + if file_url not in files_list: + print(file_url) + files_list.append(file_url) diff --git a/migration/list-videos-url.py b/migration/list-videos-url.py new file mode 100644 index 0000000000000000000000000000000000000000..d9fbc3d2dda60f4f27e7bfbad8889929080aa141 --- /dev/null +++ b/migration/list-videos-url.py @@ -0,0 +1,59 @@ +import os, re +from django.contrib.auth.models import User +from django.contrib.sites.models import Site +from pod.video.models import Video + +# cd /pod/src # GOTO pod src directoy (where is manage.py) +# python manage.py shell < /pod/REPRISE/esup-pod-api-sample/migration/list-videos-url.py > /pod/REPRISE/esup-pod-api-sample/migration/media_videos_list.txt + +# ADAPT - Domain of users and videos +site_domain_keep="mediatheque-pedagogique-test.unicaen.fr" + +# ADAPT - Videos base path +base_videos_path="/pod/media/videos" + +keep_site= Site.objects.filter(domain=site_domain_keep) + +videos_list=[] + +users=User.objects.all() +for user in users: + username=user.username + userkey=user.owner.hashkey + + if(user.owner.sites.filter(domain=site_domain_keep).exists()) : + user_videos_path=base_videos_path+"/"+userkey + if os.path.isdir(user_videos_path): + for f in os.listdir(user_videos_path): + if os.path.isdir(user_videos_path+"/"+f): + for fd in os.listdir(user_videos_path+"/"+f): + video_url="https://%s/media/videos/%s/%s/%s" % (site_domain_keep,userkey,f,fd) + if video_url not in videos_list: + print(video_url) + videos_list.append(video_url) + else: + video_url="https://%s/media/videos/%s/%s" % (site_domain_keep,userkey,f) + if video_url not in videos_list: + print(video_url) + videos_list.append(video_url) + +videos = Video.objects.filter(sites=keep_site) +for video in videos: + video_id=video.id + try: + origin_video_path = re.sub(r'^/.*/media/videos/', '', str(video.video.file)) + video_url="https://%s/media/videos/%s" % (site_domain_keep,origin_video_path) + if video_url not in videos_list: + print(video_url) + videos_list.append(video_url) + + render_video_path=os.path.dirname(str(video.video.file))+"/"+f'{video_id:04}' + if os.path.isdir(render_video_path): + file_base_path = re.sub(r'^/.*/media/videos/', '', render_video_path) + for f in os.listdir(render_video_path): + video_url="https://%s/media/videos/%s/%s" % (site_domain_keep,file_base_path,f) + if video_url not in videos_list: + print(video_url) + videos_list.append(video_url) + except FileNotFoundError: + pass diff --git a/migration/test-files-after-download.py b/migration/test-files-after-download.py new file mode 100644 index 0000000000000000000000000000000000000000..67e3f902d885423d347f6ec6f3ad66c9afd9c3fa --- /dev/null +++ b/migration/test-files-after-download.py @@ -0,0 +1,32 @@ +import os +import sys +import re + +# python /pod/REPRISE/esup-pod-api-sample/migration/test-files-after-download.py + +# ADAPT - Videos base path +base_files_path="/pod/media/files" + +# ADAPT - Input file +media_files_list="/pod/REPRISE/esup-pod-api-sample/migration/media_files_list.txt" + +print("***************** USE media_files_list : %s *************"%media_files_list) + +fmedia = open(media_files_list, 'r') +lines = fmedia.readlines() +for l in lines: + l = l.strip() + parts = re.search('^https://(.*)/media/files/(.*)/(.*)$', l, re.IGNORECASE) + domain=parts.group(1) + hashkey=parts.group(2) + filename=parts.group(3) + + hashPath=base_files_path+"/"+hashkey + testFilePath =hashPath+"/"+filename + + if not os.path.exists(testFilePath): + print("File %s not exist"%testFilePath) + + if os.stat(testFilePath).st_size == 0 : + print("File %s emtpy"%testFilePath) + diff --git a/migration/test-videos-after-download.py b/migration/test-videos-after-download.py new file mode 100644 index 0000000000000000000000000000000000000000..0d9dfda0588c633c0ffb2103f99106de81c98e5c --- /dev/null +++ b/migration/test-videos-after-download.py @@ -0,0 +1,46 @@ +import os +import sys +import re + +# python /pod/REPRISE/esup-pod-api-sample/migration/test-videos-after-download.py + +# ADAPT - Videos base path +base_videos_path="/pod/media/videos" + +# ADAPT - Input file +media_videos_list="/pod/REPRISE/esup-pod-api-sample/migration/media_videos_list.txt" + + +print("***************** USE videos_files_list : %s *************"%media_videos_list) +fvideo = open(media_videos_list, 'r') +lines = fvideo.readlines() + +for l in lines: + l = l.strip() + if re.match(r'^https://(.*)/media/videos/(.*)/(.*)/(.*)$', l): + parts = re.search('^https://(.*)/media/videos/(.*)/(.*)/(.*)$', l, re.IGNORECASE) + domain=parts.group(1) + hashkey=parts.group(2) + dirname=parts.group(3) + filename=parts.group(4) + hashPath = base_videos_path+"/"+hashkey + videoIdPath = hashPath+"/"+dirname + testFilePath = videoIdPath+"/"+filename + + if not os.path.exists(testFilePath): + print("File %s not exist"%testFilePath) + elif os.stat(testFilePath).st_size == 0 : + print("File %s emtpy"%testFilePath) + + elif re.match(r'^https://(.*)/media/videos/(.*)/(.*)$', l): + parts = re.search('^https://(.*)/media/videos/(.*)/(.*)$', l, re.IGNORECASE) + domain=parts.group(1) + hashkey=parts.group(2) + filename=parts.group(3) + hashPath = base_videos_path+"/"+hashkey + testFilePath = hashPath+"/"+filename + + if not os.path.exists(testFilePath): + print("File %s not exist"%testFilePath) + elif os.stat(testFilePath).st_size == 0 : + print("File %s emtpy"%testFilePath)