From 1cdac623831b8b16e091f767c99b42c7af973530 Mon Sep 17 00:00:00 2001 From: gabe Date: Wed, 14 Jul 2021 16:55:45 -0500 Subject: [PATCH 01/15] sets out command line options. --- adocStaticSiteGen.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 adocStaticSiteGen.py diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py new file mode 100644 index 0000000..ec72c63 --- /dev/null +++ b/adocStaticSiteGen.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python3 +#takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. + +import subprocess, sys, argparse, logging +from pathlib import Path + +logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) +#logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.DEBUG) + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument('inputDir', type=Path, help='The directory of adoc files to be copied and converted.') + parser.add_argument('-o', '--output', type=Path, help='What to name the generated directory or tar file') + parser.add_argument('-z', '--compress', action='store_true') + args=parser.parse_args() + + if args.output != None and args.compress == False: + #detect based on whether outFile has a .tar.gz filename. + if args.output.suffixes == ['.tar', '.gz']: + compress = True + else: + compress = False + else: + compress = args.compress + + if args.output == None: + outfile = Path(args.inputDir.parent).joinpath(args.inputdir.stem) + else: + outfile=args.output + + logging.info(f'outputting to {outFile}') + + return args.inputDir, outfile, compress From cbb7f9c37a52c787dd43f70c5af4bf3525bd0fb1 Mon Sep 17 00:00:00 2001 From: gabe Date: Thu, 15 Jul 2021 14:13:00 -0500 Subject: [PATCH 02/15] fixed compressed file name detection --- adocStaticSiteGen.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) mode change 100644 => 100755 adocStaticSiteGen.py diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py old mode 100644 new mode 100755 index ec72c63..52aeaac --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -1,20 +1,20 @@ #! /usr/bin/env python3 #takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. -import subprocess, sys, argparse, logging +import subprocess, sys, argparse, logging, tempfile from pathlib import Path -logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) -#logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.DEBUG) +#logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) +logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.DEBUG) def parse_arguments(): - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description='create a website directory structure by converting .adoc files in a directory strucutre to .html files.') parser.add_argument('inputDir', type=Path, help='The directory of adoc files to be copied and converted.') parser.add_argument('-o', '--output', type=Path, help='What to name the generated directory or tar file') - parser.add_argument('-z', '--compress', action='store_true') + parser.add_argument('-z', '--compress', action='store_true', help='whether to compress the resulting directory to a tar.gz file. can be usefull for scripting to transfer the site to a remote server.') args=parser.parse_args() - if args.output != None and args.compress == False: + if args.output != None and not args.compress: #detect based on whether outFile has a .tar.gz filename. if args.output.suffixes == ['.tar', '.gz']: compress = True @@ -24,10 +24,20 @@ def parse_arguments(): compress = args.compress if args.output == None: - outfile = Path(args.inputDir.parent).joinpath(args.inputdir.stem) + outFile = args.inputDir.with_name(args.inputDir.name+'_compiled') else: - outfile=args.output + outFile=args.output - logging.info(f'outputting to {outFile}') + if compress and outFile.suffixes != ['.tar', '.gz']: + logging.debug(f'outFile was {outFile}, corrected because compress flag is set.') + outFile = outFile.with_suffix('.tar.gz') - return args.inputDir, outfile, compress + logging.debug(f'inputing from {args.inputDir.resolve()}') + logging.info(f'outputting to {outFile.resolve()}') + logging.debug(f'compress is {compress}') + + if args.inputDir.resolve() == outFile.resolve(): + raise FileExistsError('output file cannot have the same path as the input file!') + + return args.inputDir, outFile, compress +parse_arguments() From 67094fc415ac7cd1840d2d037ca8f17594c4f342 Mon Sep 17 00:00:00 2001 From: gabe Date: Thu, 15 Jul 2021 17:00:39 -0500 Subject: [PATCH 03/15] first pas at the temp directory object. not tested yet. --- adocStaticSiteGen.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 52aeaac..095da0b 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -1,7 +1,7 @@ #! /usr/bin/env python3 #takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. -import subprocess, sys, argparse, logging, tempfile +import subprocess, sys, argparse, logging, tempfile, tarfile from pathlib import Path #logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) @@ -40,4 +40,22 @@ def parse_arguments(): raise FileExistsError('output file cannot have the same path as the input file!') return args.inputDir, outFile, compress + +class TmpDir: + def __init__(self, srcDir): + logging.debug('making tmp file') + self.tmpDir = tempfile.TemporaryDirectory() + self.ignorePattern = shutil.ignore_patterns('*.adoc') + shutil.copytree(srcDir, self.tmpDir, ignore = shutil.ignore_patterns('*.adoc'), symlinks=False) + + def copy_self_to(self, destDir): + shutil.copytree(self.tmpDir, destDir, symlinks=False) + + def compress_and_copy_self_to(self, destPath): + tarFile=tarfile.open(name=Path(destPath).resolve, mode='x:gz', dereference=True) + tarFile = shutil.make_archive(Path(destPath).resolve(), 'gztar', self.tmpDir) + + def cleanup(self): + tmpDir.cleanup() + parse_arguments() From 3d402e980ada45a7e3a7482d924730b82726fe08 Mon Sep 17 00:00:00 2001 From: gabe Date: Fri, 16 Jul 2021 16:04:37 -0500 Subject: [PATCH 04/15] fixed some things with the tmpfile object. --- adocStaticSiteGen.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 095da0b..a4708eb 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -45,14 +45,13 @@ class TmpDir: def __init__(self, srcDir): logging.debug('making tmp file') self.tmpDir = tempfile.TemporaryDirectory() - self.ignorePattern = shutil.ignore_patterns('*.adoc') - shutil.copytree(srcDir, self.tmpDir, ignore = shutil.ignore_patterns('*.adoc'), symlinks=False) + self.ignorePattern = shutil.ignore_patterns('*.adoc', '.git', '.gitignore') + shutil.copytree(srcDir, self.tmpDir, ignore = self.ignorePattern, symlinks=False) def copy_self_to(self, destDir): shutil.copytree(self.tmpDir, destDir, symlinks=False) def compress_and_copy_self_to(self, destPath): - tarFile=tarfile.open(name=Path(destPath).resolve, mode='x:gz', dereference=True) tarFile = shutil.make_archive(Path(destPath).resolve(), 'gztar', self.tmpDir) def cleanup(self): From 66e21b3a0bdda097cb4bd638f2f22e2d9099a74a Mon Sep 17 00:00:00 2001 From: gabe Date: Fri, 16 Jul 2021 16:37:09 -0500 Subject: [PATCH 05/15] I need a gitignore now --- .gitignore | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb0a0b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,142 @@ +#vim session files +*.vims + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ From 5cace76020aa5b1850d855ea2eb0b64cce2ff243 Mon Sep 17 00:00:00 2001 From: gabe Date: Tue, 10 Aug 2021 15:17:04 -0500 Subject: [PATCH 06/15] added first pass of compress and output. --- adocStaticSiteGen.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index a4708eb..c7110d2 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -1,7 +1,7 @@ #! /usr/bin/env python3 #takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. -import subprocess, sys, argparse, logging, tempfile, tarfile +import subprocess, sys, argparse, logging, tempfile, shutil from pathlib import Path #logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) @@ -45,16 +45,25 @@ class TmpDir: def __init__(self, srcDir): logging.debug('making tmp file') self.tmpDir = tempfile.TemporaryDirectory() + self.path = self.tmpDir.name+'/data/' self.ignorePattern = shutil.ignore_patterns('*.adoc', '.git', '.gitignore') - shutil.copytree(srcDir, self.tmpDir, ignore = self.ignorePattern, symlinks=False) + shutil.copytree(srcDir, self.path, ignore = self.ignorePattern, symlinks=False) def copy_self_to(self, destDir): - shutil.copytree(self.tmpDir, destDir, symlinks=False) + shutil.copytree(self.path, destDir, symlinks=False) def compress_and_copy_self_to(self, destPath): - tarFile = shutil.make_archive(Path(destPath).resolve(), 'gztar', self.tmpDir) + #shutil.make_archive wants destPath to be without file extentions for some godforsaken reason. + destPath = Path(destPath.with_name(destPath.name.split('.')[0])).resolve() + logging.debug(f'outputting to {destPath}') + tarFile = shutil.make_archive(destPath, 'gztar', self.path) def cleanup(self): - tmpDir.cleanup() + self.tmpDir.cleanup() -parse_arguments() +inputDir, outFile, compress = parse_arguments() +tmpdir = TmpDir(inputDir) +print(tmpdir.path) +breakpoint() +tmpdir.compress_and_copy_self_to(outFile) +tmpdir.cleanup() From 48007575aa810c458886b8fdaab527ca81c6b94b Mon Sep 17 00:00:00 2001 From: gabe venberg Date: Sun, 15 Aug 2021 10:10:25 -0500 Subject: [PATCH 07/15] added more comments! --- adocStaticSiteGen.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index c7110d2..ef1f75d 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -14,6 +14,7 @@ def parse_arguments(): parser.add_argument('-z', '--compress', action='store_true', help='whether to compress the resulting directory to a tar.gz file. can be usefull for scripting to transfer the site to a remote server.') args=parser.parse_args() + #set compress flag if args.output != None and not args.compress: #detect based on whether outFile has a .tar.gz filename. if args.output.suffixes == ['.tar', '.gz']: @@ -23,35 +24,40 @@ def parse_arguments(): else: compress = args.compress + #If outfile was not set, set it. if args.output == None: outFile = args.inputDir.with_name(args.inputDir.name+'_compiled') else: outFile=args.output + #add .tar.gz if compress is set and the outfile does not already have it. if compress and outFile.suffixes != ['.tar', '.gz']: - logging.debug(f'outFile was {outFile}, corrected because compress flag is set.') + logging.info(f'outFile was {outFile}, corrected because compress flag is set.') outFile = outFile.with_suffix('.tar.gz') + if args.inputDir.resolve() == outFile.resolve(): + raise FileExistsError('output file cannot have the same path as the input file!') + logging.debug(f'inputing from {args.inputDir.resolve()}') logging.info(f'outputting to {outFile.resolve()}') logging.debug(f'compress is {compress}') - if args.inputDir.resolve() == outFile.resolve(): - raise FileExistsError('output file cannot have the same path as the input file!') - return args.inputDir, outFile, compress +#Doing it in a tmpDir first, as some distrubutions put temp files on a ramdisk. this should speed up the operation sigificantly. class TmpDir: def __init__(self, srcDir): - logging.debug('making tmp file') self.tmpDir = tempfile.TemporaryDirectory() + logging.debug(f'making tmp file from {srcdir} at {tmpDir.name}') self.path = self.tmpDir.name+'/data/' self.ignorePattern = shutil.ignore_patterns('*.adoc', '.git', '.gitignore') shutil.copytree(srcDir, self.path, ignore = self.ignorePattern, symlinks=False) + #copy out from tmpDir (which may be in RAM, depending on distrubution) to disk def copy_self_to(self, destDir): shutil.copytree(self.path, destDir, symlinks=False) + #copy out from tmpDir (which may be in RAM, depending on distrubution) to a compressed file on disk def compress_and_copy_self_to(self, destPath): #shutil.make_archive wants destPath to be without file extentions for some godforsaken reason. destPath = Path(destPath.with_name(destPath.name.split('.')[0])).resolve() @@ -65,5 +71,6 @@ inputDir, outFile, compress = parse_arguments() tmpdir = TmpDir(inputDir) print(tmpdir.path) breakpoint() +tmpdir.copy_self_to(outFile) tmpdir.compress_and_copy_self_to(outFile) tmpdir.cleanup() From 7370b145d06abc34e7aae68daaad4028b5d5a97d Mon Sep 17 00:00:00 2001 From: gabe Date: Wed, 1 Sep 2021 17:42:19 -0500 Subject: [PATCH 08/15] added start of recursive directory finding. commiting before testing code refactor. --- adocStaticSiteGen.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index ef1f75d..3715273 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -1,7 +1,7 @@ #! /usr/bin/env python3 #takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. -import subprocess, sys, argparse, logging, tempfile, shutil +import subprocess, sys, argparse, logging, tempfile, shutil, os from pathlib import Path #logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) @@ -48,7 +48,7 @@ def parse_arguments(): class TmpDir: def __init__(self, srcDir): self.tmpDir = tempfile.TemporaryDirectory() - logging.debug(f'making tmp file from {srcdir} at {tmpDir.name}') + logging.debug(f'making tmp file from {srcDir} at {self.tmpDir.name}') self.path = self.tmpDir.name+'/data/' self.ignorePattern = shutil.ignore_patterns('*.adoc', '.git', '.gitignore') shutil.copytree(srcDir, self.path, ignore = self.ignorePattern, symlinks=False) @@ -67,10 +67,17 @@ class TmpDir: def cleanup(self): self.tmpDir.cleanup() +#pass an empty list to start this. It calls itself recursively +def find_paths_to_convert(inputDir, pathList): + with os.scandir(inputDir) as it: + for path in it: + logging.debug(f'found {path.path}') + if path.is_dir(): + logging.debug(f'{path.path} is directory, recursing') + find_paths_to_convert(path, pathList) + inputDir, outFile, compress = parse_arguments() -tmpdir = TmpDir(inputDir) -print(tmpdir.path) +tmpDir=TmpDir(inputDir) breakpoint() -tmpdir.copy_self_to(outFile) -tmpdir.compress_and_copy_self_to(outFile) -tmpdir.cleanup() +tmpDir.cleanup() +find_paths_to_convert(inputDir, []) From 8dc4b62cf43e7bfb701e065559562eb121a4ed62 Mon Sep 17 00:00:00 2001 From: gabe Date: Thu, 2 Sep 2021 19:43:00 -0500 Subject: [PATCH 09/15] added function to find files needing conversion. --- adocStaticSiteGen.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 3715273..863400b 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -1,7 +1,7 @@ #! /usr/bin/env python3 #takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. -import subprocess, sys, argparse, logging, tempfile, shutil, os +import subprocess, sys, argparse, logging, tempfile, shutil, os, re from pathlib import Path #logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) @@ -75,9 +75,14 @@ def find_paths_to_convert(inputDir, pathList): if path.is_dir(): logging.debug(f'{path.path} is directory, recursing') find_paths_to_convert(path, pathList) + elif path.is_file() and re.match('^.*\.adoc$', path.name): + logging.debug(f'adding {path.name} to pathList') + pathList.append(Path(path.path)) + return pathList -inputDir, outFile, compress = parse_arguments() -tmpDir=TmpDir(inputDir) +inFile, outFile, compress = parse_arguments() +os.chdir(inFile) +tmpDir=TmpDir('./') breakpoint() tmpDir.cleanup() -find_paths_to_convert(inputDir, []) +print(find_paths_to_convert('./', [])) From ceeb38a6676aa102eafc9676f00b223d95f8bdb2 Mon Sep 17 00:00:00 2001 From: gabe Date: Sat, 4 Sep 2021 12:58:10 -0500 Subject: [PATCH 10/15] added conversion function. Need to do speedtest to see whether this method is faster or slower than calling unix find. --- adocStaticSiteGen.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 863400b..ec550ce 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -80,9 +80,21 @@ def find_paths_to_convert(inputDir, pathList): pathList.append(Path(path.path)) return pathList +#simple wrapper around the asciidoctor cli. +def convert_file(inDir, outDir, inFile): + logging.debug(f'converting {inFile} from directory {inDir} to directory {outDir}') + try: + subprocess.run(['asciidoctor', f'--base-dir={inDir}', f'--source-dir={inDir}', f'--destination-dir={outDir}', inFile], check=True) + except Exception as e: + logging.error(f'could not convert {inFile}!') + logging.error(f'stdErr was {e.stderr}') + logging.error(f'stdOut was {e.stdout}') + inFile, outFile, compress = parse_arguments() os.chdir(inFile) tmpDir=TmpDir('./') +pathsToConvert=find_paths_to_convert('./', []) +for i in pathsToConvert: + convert_file('./', tmpDir.path, i) breakpoint() tmpDir.cleanup() -print(find_paths_to_convert('./', [])) From 1e05e48c7bf4480abcbf72ba15f512f7e173acb1 Mon Sep 17 00:00:00 2001 From: gabe Date: Sat, 4 Sep 2021 13:11:31 -0500 Subject: [PATCH 11/15] cleaned up some logging --- adocStaticSiteGen.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index ec550ce..587c894 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -55,13 +55,14 @@ class TmpDir: #copy out from tmpDir (which may be in RAM, depending on distrubution) to disk def copy_self_to(self, destDir): + logging.debug(f'outputting to {destPath}') shutil.copytree(self.path, destDir, symlinks=False) #copy out from tmpDir (which may be in RAM, depending on distrubution) to a compressed file on disk def compress_and_copy_self_to(self, destPath): #shutil.make_archive wants destPath to be without file extentions for some godforsaken reason. destPath = Path(destPath.with_name(destPath.name.split('.')[0])).resolve() - logging.debug(f'outputting to {destPath}') + logging.debug(f'compressing to {destPath}') tarFile = shutil.make_archive(destPath, 'gztar', self.path) def cleanup(self): @@ -82,8 +83,10 @@ def find_paths_to_convert(inputDir, pathList): #simple wrapper around the asciidoctor cli. def convert_file(inDir, outDir, inFile): + logging.info(f'converting {Path(inFile).resolve()}') logging.debug(f'converting {inFile} from directory {inDir} to directory {outDir}') try: + #the destdir can be used instead of destfile in order to preserve the directory structure relative to the base dir. really useful. subprocess.run(['asciidoctor', f'--base-dir={inDir}', f'--source-dir={inDir}', f'--destination-dir={outDir}', inFile], check=True) except Exception as e: logging.error(f'could not convert {inFile}!') From 74796c804e638a9195b54b019944344d43640124 Mon Sep 17 00:00:00 2001 From: gabe Date: Sun, 5 Sep 2021 00:07:48 -0500 Subject: [PATCH 12/15] finishing touches on basic functionality. Misc cleanup, more comments, and finishing the basic functionality. Just need to implement issue #4. Possible early release candidate? --- adocStaticSiteGen.py | 58 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 587c894..16d0dc1 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -4,11 +4,11 @@ import subprocess, sys, argparse, logging, tempfile, shutil, os, re from pathlib import Path -#logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) -logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.DEBUG) +logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) +#logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.DEBUG) def parse_arguments(): - parser = argparse.ArgumentParser(description='create a website directory structure by converting .adoc files in a directory strucutre to .html files.') + parser=argparse.ArgumentParser(description='create a website directory structure by converting .adoc files in a directory strucutre to .html files.') parser.add_argument('inputDir', type=Path, help='The directory of adoc files to be copied and converted.') parser.add_argument('-o', '--output', type=Path, help='What to name the generated directory or tar file') parser.add_argument('-z', '--compress', action='store_true', help='whether to compress the resulting directory to a tar.gz file. can be usefull for scripting to transfer the site to a remote server.') @@ -18,22 +18,22 @@ def parse_arguments(): if args.output != None and not args.compress: #detect based on whether outFile has a .tar.gz filename. if args.output.suffixes == ['.tar', '.gz']: - compress = True + compress=True else: - compress = False + compress=False else: - compress = args.compress + compress=args.compress #If outfile was not set, set it. if args.output == None: - outFile = args.inputDir.with_name(args.inputDir.name+'_compiled') + outFile=args.inputDir.with_name(args.inputDir.name+'_compiled').resolve() else: - outFile=args.output + outFile=args.output.resovle() #add .tar.gz if compress is set and the outfile does not already have it. if compress and outFile.suffixes != ['.tar', '.gz']: logging.info(f'outFile was {outFile}, corrected because compress flag is set.') - outFile = outFile.with_suffix('.tar.gz') + outFile=outFile.with_suffix('.tar.gz').resolve() if args.inputDir.resolve() == outFile.resolve(): raise FileExistsError('output file cannot have the same path as the input file!') @@ -42,28 +42,28 @@ def parse_arguments(): logging.info(f'outputting to {outFile.resolve()}') logging.debug(f'compress is {compress}') - return args.inputDir, outFile, compress + return args.inputDir.resolve(), outFile, compress #Doing it in a tmpDir first, as some distrubutions put temp files on a ramdisk. this should speed up the operation sigificantly. class TmpDir: def __init__(self, srcDir): - self.tmpDir = tempfile.TemporaryDirectory() + self.tmpDir=tempfile.TemporaryDirectory() logging.debug(f'making tmp file from {srcDir} at {self.tmpDir.name}') - self.path = self.tmpDir.name+'/data/' - self.ignorePattern = shutil.ignore_patterns('*.adoc', '.git', '.gitignore') - shutil.copytree(srcDir, self.path, ignore = self.ignorePattern, symlinks=False) + self.path=self.tmpDir.name+'/'+Path(srcDir).resolve().name + self.ignorePattern=shutil.ignore_patterns('*.adoc', '.git', '.gitignore') + shutil.copytree(srcDir, self.path, ignore=self.ignorePattern, symlinks=False) #copy out from tmpDir (which may be in RAM, depending on distrubution) to disk - def copy_self_to(self, destDir): - logging.debug(f'outputting to {destPath}') - shutil.copytree(self.path, destDir, symlinks=False) + def copy_self_to(self, destPath): + logging.debug(f'outputting to {Path(destPath).resolve()}') + shutil.copytree(self.path, destPath, symlinks=False) #copy out from tmpDir (which may be in RAM, depending on distrubution) to a compressed file on disk def compress_and_copy_self_to(self, destPath): #shutil.make_archive wants destPath to be without file extentions for some godforsaken reason. - destPath = Path(destPath.with_name(destPath.name.split('.')[0])).resolve() - logging.debug(f'compressing to {destPath}') - tarFile = shutil.make_archive(destPath, 'gztar', self.path) + destPath=Path(destPath.with_name(destPath.name.split('.')[0])).resolve() + logging.debug(f'compressing to {Path(destPath).resolve()} from {Path(self.path).parent}') + tarFile=shutil.make_archive(destPath, 'gztar', Path(self.path).parent) def cleanup(self): self.tmpDir.cleanup() @@ -87,17 +87,29 @@ def convert_file(inDir, outDir, inFile): logging.debug(f'converting {inFile} from directory {inDir} to directory {outDir}') try: #the destdir can be used instead of destfile in order to preserve the directory structure relative to the base dir. really useful. - subprocess.run(['asciidoctor', f'--base-dir={inDir}', f'--source-dir={inDir}', f'--destination-dir={outDir}', inFile], check=True) + subprocess.run(['asciidoctor', + #specifies the source directory root. + f'--source-dir={inDir}', + #Destination dir. It takes the file from the subtree --source-dir and puts it in the equivilant location in the subtree --destination-dir. (talking about filesystem subtrees). + f'--destination-dir={outDir}', + inFile], + check=True) except Exception as e: logging.error(f'could not convert {inFile}!') logging.error(f'stdErr was {e.stderr}') logging.error(f'stdOut was {e.stdout}') -inFile, outFile, compress = parse_arguments() +inFile, outFile, compress=parse_arguments() os.chdir(inFile) tmpDir=TmpDir('./') pathsToConvert=find_paths_to_convert('./', []) + for i in pathsToConvert: convert_file('./', tmpDir.path, i) -breakpoint() + +if compress: + tmpDir.compress_and_copy_self_to(outFile) +else: + tmpDir.copy_self_to(outFile) + tmpDir.cleanup() From 7ad4eaa71bfa727c7a484633b8efb40d490b5bb7 Mon Sep 17 00:00:00 2001 From: gabe Date: Sun, 5 Sep 2021 11:02:51 -0500 Subject: [PATCH 13/15] wrapped things in if __name__ == '__main__'. --- adocStaticSiteGen.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 16d0dc1..7d73d53 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -99,17 +99,18 @@ def convert_file(inDir, outDir, inFile): logging.error(f'stdErr was {e.stderr}') logging.error(f'stdOut was {e.stdout}') -inFile, outFile, compress=parse_arguments() -os.chdir(inFile) -tmpDir=TmpDir('./') -pathsToConvert=find_paths_to_convert('./', []) +if __name__ == '__main__': + inFile, outFile, compress=parse_arguments() + os.chdir(inFile) + tmpDir=TmpDir('./') + pathsToConvert=find_paths_to_convert('./', []) -for i in pathsToConvert: - convert_file('./', tmpDir.path, i) + for i in pathsToConvert: + convert_file('./', tmpDir.path, i) -if compress: - tmpDir.compress_and_copy_self_to(outFile) -else: - tmpDir.copy_self_to(outFile) + if compress: + tmpDir.compress_and_copy_self_to(outFile) + else: + tmpDir.copy_self_to(outFile) -tmpDir.cleanup() + tmpDir.cleanup() From b080c53ed5f383dfb32beb03578e752edba4146a Mon Sep 17 00:00:00 2001 From: gabe Date: Sun, 5 Sep 2021 13:30:05 -0500 Subject: [PATCH 14/15] made default output file be in current directory. fixes #6. --- adocStaticSiteGen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index 7d73d53..c2a45e7 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -26,7 +26,8 @@ def parse_arguments(): #If outfile was not set, set it. if args.output == None: - outFile=args.inputDir.with_name(args.inputDir.name+'_compiled').resolve() + baseName=args.inputDir.with_name(args.inputDir.name+'_compiled').name + outFile=Path(os.getcwd()).joinpath(baseName) else: outFile=args.output.resovle() From e37f9e4217176f67cf56428ee3bebab6b1f3a528 Mon Sep 17 00:00:00 2001 From: gabe Date: Tue, 7 Sep 2021 11:54:14 -0500 Subject: [PATCH 15/15] changed homade function to glob based fucntion. Now use globs for find_paths_to_convert. --- adocStaticSiteGen.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/adocStaticSiteGen.py b/adocStaticSiteGen.py index c2a45e7..cbeee9d 100755 --- a/adocStaticSiteGen.py +++ b/adocStaticSiteGen.py @@ -1,7 +1,7 @@ #! /usr/bin/env python3 #takes directory, converts all .adoc files to html files, copying the resulting html files to an identical directory strucuture, and copies over all non .adoc files unchanged. Optionally outputs as a tar.gz file. -import subprocess, sys, argparse, logging, tempfile, shutil, os, re +import subprocess, sys, argparse, logging, tempfile, shutil, os, glob from pathlib import Path logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.INFO) @@ -69,18 +69,9 @@ class TmpDir: def cleanup(self): self.tmpDir.cleanup() -#pass an empty list to start this. It calls itself recursively -def find_paths_to_convert(inputDir, pathList): - with os.scandir(inputDir) as it: - for path in it: - logging.debug(f'found {path.path}') - if path.is_dir(): - logging.debug(f'{path.path} is directory, recursing') - find_paths_to_convert(path, pathList) - elif path.is_file() and re.match('^.*\.adoc$', path.name): - logging.debug(f'adding {path.name} to pathList') - pathList.append(Path(path.path)) - return pathList +#works on the current working directory +def find_paths_to_convert(fileNameGlob): + return glob.glob(f'**/{fileNameGlob}', recursive=True) #simple wrapper around the asciidoctor cli. def convert_file(inDir, outDir, inFile): @@ -104,7 +95,7 @@ if __name__ == '__main__': inFile, outFile, compress=parse_arguments() os.chdir(inFile) tmpDir=TmpDir('./') - pathsToConvert=find_paths_to_convert('./', []) + pathsToConvert=find_paths_to_convert('*.adoc') for i in pathsToConvert: convert_file('./', tmpDir.path, i)