Repository navigation
Expand file tree
/
Copy pathpythontex3.py
More file actions
2882 lines (2682 loc) · 139 KB
/
Copy pathpythontex3.py
File metadata and controls
2882 lines (2682 loc) · 139 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
This is the main PythonTeX script. It should be launched via pythontex.py.
Two versions of this script are provided. One, with name ending in "2", runs
under Python 2.7. The other, with name ending in "3", runs under Python 3.2+.
This script needs to be able to import pythontex_engines.py; in general it
should be in the same directory.
Licensed under the BSD 3-Clause License:
Copyright (c) 2012-2020, Geoffrey M. Poore
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the <organization> nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
# Imports
#// Python 2
#from __future__ import absolute_import
#from __future__ import division
#from __future__ import print_function
#from __future__ import unicode_literals
#\\ End Python 2
import sys
import os
import argparse
import codecs
import time
from hashlib import sha1
from collections import defaultdict, OrderedDict, namedtuple
from re import match, sub, search
import subprocess
import multiprocessing
from pygments.styles import get_all_styles
from pythontex_engines import *
import textwrap
import platform
import itertools
if sys.version_info[0] == 2:
try:
import cPickle as pickle
except:
import pickle
from io import open
else:
import pickle
# Script parameters
# Version
__version__ = '0.18dev'
class Pytxcode(object):
def __init__(self, data, gobble):
self.delims, self.code = data.split('#\n', 1)
self.family, self.session, self.restart, self.instance, self.command, self.context, self.args_run, self.args_prettyprint, self.input_file, self.line = self.delims.split('#')
self.instance_int = int(self.instance)
self.line_int = int(self.line)
self.key_run = self.family + '#' + self.session + '#' + self.restart
self.key_typeset = self.key_run + '#' + self.instance
self.hashable_delims_run = self.key_typeset + '#' + self.command + '#' + self.context + '#' + self.args_run
self.hashable_delims_typeset = self.key_typeset + '#' + self.command + '#' + self.context + '#' + self.args_run
if len(self.command) > 1:
self.is_inline = False
# Environments start on the next line
self.line_int += 1
self.line = str(self.line_int)
else:
self.is_inline = True
self.is_extfile = True if self.session.startswith('EXT:') else False
if self.is_extfile:
self.extfile = os.path.expanduser(os.path.normcase(self.session.replace('EXT:', '', 1)))
self.key_typeset = self.key_typeset.replace('EXT:', '')
self.is_cc = True if self.family.startswith('CC:') else False
self.is_pyg = True if self.family.startswith('PYG') else False
self.is_verb = True if self.restart.endswith('verb') else False
if self.is_cc:
self.instance += 'CC'
self.cc_type, self.cc_pos = self.family.split(':')[1:]
if self.is_verb or self.is_pyg or self.is_cc:
self.is_cons = False
else:
self.is_cons = engine_dict[self.family].console
self.is_code = False if self.is_verb or self.is_pyg or self.is_cc or self.is_cons else True
if self.command in ('c', 'code') or (self.command == 'i' and not self.is_cons):
self.is_typeset = False
else:
self.is_typeset = True
if gobble == 'auto':
self.code = textwrap.dedent(self.code)
self.sub_template = None
def process_argv(data, temp_data):
'''
Process command line options using the argparse module.
Most options are passed via the file of code, rather than via the command
line.
'''
# Create a command line argument parser
parser = argparse.ArgumentParser()
parser.add_argument('TEXNAME',
help='LaTeX file, with or without .tex extension')
parser.add_argument('--version', action='version',
version='PythonTeX {0}'.format(data['version']))
parser.add_argument('--encoding', default='UTF-8',
help='encoding for all text files (see codecs module for encodings)')
parser.add_argument('--error-exit-code', default='true',
choices=('true', 'false'),
help='return exit code of 1 if there are errors (not desirable with some TeX editors and workflows)')
group_run = parser.add_mutually_exclusive_group()
group_run.add_argument('--runall', nargs='?', default='false',
const='true', choices=('true', 'false'),
help='run ALL code; equivalent to package option')
group_run.add_argument('--rerun', default='errors',
choices=('never', 'modified', 'errors', 'warnings', 'always'),
help='set conditions for rerunning code; equivalent to package option')
parser.add_argument('--hashdependencies', nargs='?', default='false',
const='true', choices=('true', 'false'),
help='hash dependencies (such as external data) to check for modification, rather than using mtime; equivalent to package option')
parser.add_argument('-j', '--jobs', metavar='N', default=None, type=int,
help='Allow N jobs at once; defaults to cpu_count().')
parser.add_argument('-v', '--verbose', default=False, action='store_true',
help='verbose output')
parser.add_argument('--interpreter', default=None, help='set a custom interpreter; argument should be in the form "<interpreter>:<command>, <interp>:<cmd>, ..." where <interpreter> is "python", "ruby", etc., and <command> is the command for invoking the interpreter; argument may also be in the form of a Python dictionary')
group_debug = parser.add_mutually_exclusive_group()
group_debug.add_argument('--debug', nargs='?', default=None,
const='default',
metavar='<family>:<session>:<restart>',
help='Run the specified session (or default session) with the default debugger, if available. If there is only one session, it need not be specified. If the session name is unambiguous, it is sufficient. The full <family>:<session>:<restart> (for example, py:default:default) is only needed when the session name alone would be ambiguous.')
group_debug.add_argument('--interactive', nargs='?', default=None,
const='default',
metavar='<family>:<session>:<restart>',
help='Run the specified session (or default session) in interactive mode. If there is only one session, it need not be specified. If the session name is unambiguous, it is sufficient. The full <family>:<session>:<restart> (for example, py:default:default) is only needed when the session name alone would be ambiguous.')
args = parser.parse_args()
# Store the parsed argv in data and temp_data
data['encoding'] = args.encoding
if args.error_exit_code == 'true':
temp_data['error_exit_code'] = True
else:
temp_data['error_exit_code'] = False
# runall can be mapped onto rerun, so both are stored under rerun
if args.runall == 'true':
temp_data['rerun'] = 'always'
else:
temp_data['rerun'] = args.rerun
# hashdependencies need only be in temp_data, since changing it would
# change hashes (hashes of mtime vs. file contents)
if args.hashdependencies == 'true':
temp_data['hashdependencies'] = True
else:
temp_data['hashdependencies'] = False
if args.jobs is None:
try:
jobs = multiprocessing.cpu_count()
except NotImplementedError:
jobs = 1
temp_data['jobs'] = jobs
else:
temp_data['jobs'] = args.jobs
temp_data['verbose'] = args.verbose
temp_data['debug'] = args.debug
temp_data['interactive'] = args.interactive
# Update interpreter_dict based on interpreter
set_python_interpreter = False
if args.interpreter is not None:
interp_list = args.interpreter.lstrip('{').rstrip('}').split(',')
for interp in interp_list:
if interp:
try:
k, v = interp.split(':', 1)
k = k.strip(' \'"')
v = v.strip(' \'"')
interpreter_dict[k] = v
if k == 'python':
set_python_interpreter = True
except:
print('Invalid --interpreter argument')
return sys.exit(2)
# If the Python interpreter wasn't set, then try to set an appropriate
# default value, based on how PythonTeX was launched (pythontex.py,
# pythontex2.py, or pythontex3.py).
if not set_python_interpreter:
if temp_data['python'] == 2:
if platform.system() == 'Windows':
try:
subprocess.check_output(['py', '--version'])
interpreter_dict['python'] = 'py -2'
except:
msg = '''
* PythonTeX error:
You have launched PythonTeX using pythontex{0}.py
directly. This should only be done when you want
to use Python version {0}, but have a different
version installed as the default. (Otherwise, you
should start PythonTeX with pythontex.py.) For
this to work correctly, you should install Python
version 3.3+, which has a Windows wrapper (py) that
PythonTeX can use to run the correct version of
Python. If you do not want to install Python 3.3+,
you can also use the --interpreter command-line
option to tell PythonTeX how to access the version
of Python you wish to use.
'''.format(temp_data['python'])
print(textwrap.dedent(msg[1:]))
return sys.exit(2)
else:
interpreter_dict['python'] = 'python2'
elif temp_data['python'] == 3:
if platform.system() == 'Windows':
try:
subprocess.check_output(['py', '--version'])
interpreter_dict['python'] = 'py -3'
except:
msg = '''
* PythonTeX error:
You have launched PythonTeX using pythontex{0}.py
directly. This should only be done when you want
to use Python version {0}, but have a different
version installed as the default. (Otherwise, you
should start PythonTeX with pythontex.py.) For
this to work correctly, you should install Python
version 3.3+, which has a Windows wrapper (py) that
PythonTeX can use to run the correct version of
Python. If you do not want to install Python 3.3+,
you can also use the --interpreter command-line
option to tell PythonTeX how to access the version
of Python you wish to use.
'''.format(temp_data['python'])
print(textwrap.dedent(msg[1:]))
return sys.exit(2)
else:
interpreter_dict['python'] = 'python3'
if args.TEXNAME is not None:
# Determine if we a dealing with just a filename, or a name plus
# path. If there's a path, we need to make the document directory
# the current working directory.
dir, raw_jobname = os.path.split(args.TEXNAME)
dir = os.path.expanduser(os.path.normcase(dir))
if dir:
os.chdir(dir)
sys.path.append(dir)
# If necessary, strip off an extension to find the raw jobname that
# corresponds to the .pytxcode.
if not os.path.exists(raw_jobname + '.pytxcode'):
raw_jobname = raw_jobname.rsplit('.', 1)[0]
if not os.path.exists(raw_jobname + '.pytxcode'):
print('* PythonTeX error')
print(' Code file ' + raw_jobname + '.pytxcode does not exist.')
print(' Run LaTeX to create it.')
return sys.exit(1)
# We need a "sanitized" version of the jobname, with spaces and
# asterisks replaced with hyphens. This is done to avoid TeX issues
# with spaces in file names, paralleling the approach taken in
# pythontex.sty. From now on, we will use the sanitized version every
# time we create a file that contains the jobname string. The raw
# version will only be used in reference to pre-existing files created
# on the TeX side, such as the .pytxcode file.
jobname = raw_jobname.replace(' ', '-').replace('"', '').replace('*', '-')
# Store the results in data
data['raw_jobname'] = raw_jobname
data['jobname'] = jobname
# We need to check to make sure that the "sanitized" jobname doesn't
# lead to a collision with a file that already has that name, so that
# two files attempt to use the same PythonTeX folder.
#
# If <jobname>.<ext> and <raw_jobname>.<ext> both exist, where <ext>
# is a common LaTeX extension, we exit. We operate under the
# assumption that there should be only a single file <jobname> in the
# document root directory that has a common LaTeX extension. That
# could be false, but if so, the user probably has worse things to
# worry about than a potential PythonTeX output collision.
# If <jobname>* and <raw_jobname>* both exist, we issue a warning but
# attempt to proceed.
if jobname != raw_jobname:
resolved = False
for ext in ('.tex', '.ltx', '.dtx'):
if os.path.isfile(raw_jobname + ext):
if os.path.isfile(jobname + ext):
print('* PythonTeX error')
print(' Directory naming collision between the following files:')
print(' ' + raw_jobname + ext)
print(' ' + jobname + ext)
return sys.exit(1)
else:
resolved = True
break
if not resolved:
ls = os.listdir('.')
for file in ls:
if file.startswith(jobname):
print('* PythonTeX warning')
print(' Potential directory naming collision between the following names:')
print(' ' + raw_jobname)
print(' ' + jobname + '*')
print(' Attempting to proceed.')
temp_data['warnings'] += 1
break
def load_code_get_settings(data, temp_data):
'''
Load the code file, preprocess the code, and extract the settings.
'''
# Bring in the .pytxcode file as a single string
raw_jobname = data['raw_jobname']
encoding = data['encoding']
# The error checking here is a little redundant
if os.path.isfile(raw_jobname + '.pytxcode'):
f = open(raw_jobname + '.pytxcode', 'r', encoding=encoding)
pytxcode = f.read()
f.close()
else:
print('* PythonTeX error')
print(' Code file ' + raw_jobname + '.pytxcode does not exist.')
print(' Run LaTeX to create it.')
return sys.exit(1)
# Split code and settings
try:
pytxcode, pytxsettings = pytxcode.rsplit('=>PYTHONTEX:SETTINGS#', 1)
except:
print('The .pytxcode file appears to have an outdated format or be invalid')
print('Run LaTeX to make sure the file is current')
return sys.exit(1)
# Prepare to process settings
#
# Create a dict for storing settings.
settings = {}
# Create a dict for storing Pygments settings.
# Each dict entry will itself be a dict.
pygments_settings = defaultdict(dict)
# Create a dict of processing functions, and generic processing functions
settings_func = dict()
def set_kv_data(k, v):
if v == 'true':
settings[k] = True
elif v == 'false':
settings[k] = False
else:
settings[k] = v
# Need a function for when assignment is only needed if not default value
def set_kv_temp_data_if_not_default(k, v):
if v != 'default':
if v == 'true':
temp_data[k] = True
elif v == 'false':
temp_data[k] = False
else:
temp_data[k] = v
def set_kv_data_fvextfile(k, v):
# Error checking on TeX side should be enough, but be careful anyway
try:
v = int(v)
except ValueError:
print('* PythonTeX error')
print(' Unable to parse package option fvextfile.')
return sys.exit(1)
if v < 0:
settings[k] = sys.maxsize
elif v == 0:
settings[k] = 1
print('* PythonTeX warning')
print(' Invalid value for package option fvextfile.')
temp_data['warnings'] += 1
else:
settings[k] = v
def set_kv_pygments(k, v):
family, lexer_opts, options = v.replace(' ','').split('|')
lexer = None
lex_dict = {}
opt_dict = {}
if lexer_opts:
for l in lexer_opts.split(','):
if '=' in l:
k, v = l.split('=', 1)
if k == 'lexer':
lexer = l
else:
lex_dict[k] = v
else:
lexer = l
if options:
for o in options.split(','):
if '=' in o:
k, v = o.split('=', 1)
if v in ('true', 'True'):
v = True
elif v in ('false', 'False'):
v = False
else:
k = option
v = True
opt_dict[k] = v
if family != ':GLOBAL':
if 'lexer' in pygments_settings[':GLOBAL']:
lexer = pygments_settings[':GLOBAL']['lexer']
lex_dict.update(pygments_settings[':GLOBAL']['lexer_options'])
opt_dict.update(pygments_settings[':GLOBAL']['formatter_options'])
if 'style' not in opt_dict:
opt_dict['style'] = 'default'
opt_dict['commandprefix'] = 'PYG' + opt_dict['style']
if lexer is not None:
pygments_settings[family]['lexer'] = lexer
pygments_settings[family]['lexer_options'] = lex_dict
pygments_settings[family]['formatter_options'] = opt_dict
settings_func['version'] = set_kv_data
settings_func['outputdir'] = set_kv_data
settings_func['workingdir'] = set_kv_data
settings_func['workingdirset'] = set_kv_data
settings_func['gobble'] = set_kv_data
settings_func['rerun'] = set_kv_temp_data_if_not_default
settings_func['hashdependencies'] = set_kv_temp_data_if_not_default
settings_func['makestderr'] = set_kv_data
settings_func['stderrfilename'] = set_kv_data
settings_func['keeptemps'] = set_kv_data
settings_func['pyfuture'] = set_kv_data
settings_func['pyconfuture'] = set_kv_data
settings_func['pygments'] = set_kv_data
settings_func['fvextfile'] = set_kv_data_fvextfile
settings_func['pygglobal'] = set_kv_pygments
settings_func['pygfamily'] = set_kv_pygments
settings_func['pyconbanner'] = set_kv_data
settings_func['pyconfilename'] = set_kv_data
settings_func['depythontex'] = set_kv_data
# Process settings
for line in pytxsettings.split('\n'):
if line:
key, val = line.split('=', 1)
try:
settings_func[key](key, val)
except KeyError:
print('* PythonTeX warning')
print(' Unknown option "' + key + '"')
temp_data['warnings'] += 1
# Check for compatility between the .pytxcode and the script
if 'version' not in settings or settings['version'] != data['version']:
print('* PythonTeX error')
print(' The version of the PythonTeX scripts does not match the last code')
print(' saved by the document--run LaTeX to create an updated version.\n')
sys.exit(1)
# Store all results that haven't already been stored.
data['settings'] = settings
data['pygments_settings'] = pygments_settings
# Create a tuple of vital quantities that invalidate old saved data
# Don't need to include outputdir, because if that changes, no old output
# fvextfile could be checked on a case-by-case basis, which would result
# in faster output, but that would involve a good bit of additional
# logic, which probably isn't worth it for a feature that will rarely be
# changed.
data['vitals'] = (data['version'], data['encoding'],
settings['gobble'], settings['fvextfile'])
# Create tuples of vital quantities
data['code_vitals'] = (settings['workingdir'], settings['keeptemps'],
settings['makestderr'], settings['stderrfilename'])
data['cons_vitals'] = (settings['workingdir'])
data['typeset_vitals'] = ()
# Pass any customizations to types
for k in engine_dict:
engine_dict[k].customize(pyfuture=settings['pyfuture'],
pyconfuture=settings['pyconfuture'],
pyconbanner=settings['pyconbanner'],
pyconfilename=settings['pyconfilename'])
# Store code
# Do this last, so that Pygments settings are available
if pytxcode.startswith('=>PYTHONTEX#'):
gobble = settings['gobble']
temp_data['pytxcode'] = [Pytxcode(c, gobble) for c in pytxcode.split('=>PYTHONTEX#')[1:]]
else:
temp_data['pytxcode'] = []
def set_upgrade_compatibility(data, old, temp_data):
'''
When upgrading, modify settings to maintain backward compatibility when
possible and important
'''
if (old['version'].startswith('v') and
not data['settings']['workingdirset'] and
data['settings']['outputdir'] != '.'):
old['compatibility'] = '0.13'
do_upgrade_compatibility(data, old, temp_data)
def do_upgrade_compatibility(data, old_data, temp_data):
if 'compatibility' in old_data:
c = old_data['compatibility']
if (c == '0.13' and not data['settings']['workingdirset'] and
data['settings']['outputdir'] != '.'):
data['compatibility'] = c
data['settings']['workingdir'] = data['settings']['outputdir']
msg = '''
**** PythonTeX upgrade message ****
Beginning with v0.14, the default working directory is the document
directory rather than the output directory. PythonTeX has detected
that you have been using the output directory as the working directory.
It will continue to use the output directory for now. To keep your
current settings long-term and avoid seeing this message in the future,
add the following command to the preamble of your document, right after
the "\\usepackage{pythontex}": "\setpythontexworkingdir{<outputdir>}".
If you wish to continue with the new settings instead, simply delete
the file with extension .pkl in the output directory, and run PythonTeX.
**** End PythonTeX upgrade message ****
'''
temp_data['upgrade_message'] = textwrap.dedent(msg)
def get_old_data(data, old_data, temp_data):
'''
Load data from the last run, if it exists, into the dict old_data.
Determine the path to the PythonTeX scripts, either by using a previously
found, saved path or via kpsewhich.
The old data is used for determining when PythonTeX has been upgraded,
when any settings have changed, when code has changed (via hashes), and
what files may need to be cleaned up. The location of the PythonTeX
scripts is needed so that they can be imported by the scripts created by
PythonTeX. The location of the scripts is confirmed even if they were
previously located, to make sure that the path is still valid. Finding
the scripts depends on having a TeX installation that includes the
Kpathsea library (TeX Live and MiKTeX, possibly others).
All code that relies on old_data is written based on the assumption that
if old_data exists and has the current PythonTeX version, then it
contains all needed information. Thus, all code relying on old_data must
check that it was loaded and that it has the current version. If not,
code should adapt gracefully.
'''
# Create a string containing the name of the data file
pythontex_data_file = os.path.expanduser(os.path.normcase(os.path.join(data['settings']['outputdir'], 'pythontex_data.pkl')))
# Load the old data if it exists (read as binary pickle)
if os.path.isfile(pythontex_data_file):
f = open(pythontex_data_file, 'rb')
old = pickle.load(f)
f.close()
# Check for compabilility
if 'vitals' in old and data['vitals'] == old['vitals']:
temp_data['loaded_old_data'] = True
old_data.update(old)
do_upgrade_compatibility(data, old_data, temp_data)
else:
if 'version' in old and old['version'] != data['version']:
set_upgrade_compatibility(data, old, temp_data)
temp_data['loaded_old_data'] = False
# Clean up all old files
if 'files' in old:
for key in old['files']:
for f in old['files'][key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
if 'pygments_files' in old:
for key in old['pygments_files']:
for f in old['pygments_files'][key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
else:
temp_data['loaded_old_data'] = False
# Set the utilspath
# Assume that if the utils aren't in the same location as
# `pythontex.py`, then they are somewhere else on `sys.path` that
# will always be available (for example, installed as a Python module),
# and thus specifying a path isn't necessary.
if os.path.isfile(os.path.join(sys.path[0], 'pythontex_utils.py')):
# Need the path with forward slashes, so escaping isn't necessary
data['utilspath'] = sys.path[0].replace('\\', '/')
else:
data['utilspath'] = ''
def modified_dependencies(key, data, old_data, temp_data):
hashdependencies = temp_data['hashdependencies']
if key not in old_data['dependencies']:
return False
else:
old_dep_hash_dict = old_data['dependencies'][key]
workingdir = data['settings']['workingdir']
for dep in old_dep_hash_dict.keys():
# We need to know if the path is relative (based off the
# working directory) or absolute. We can't use
# os.path.isabs() alone for determining the distinction,
# because we must take into account the possibility of an
# initial ~ (tilde) standing for the home directory.
dep_file = os.path.expanduser(os.path.normcase(dep))
if not os.path.isabs(dep_file):
dep_file = os.path.expanduser(os.path.normcase(os.path.join(workingdir, dep_file)))
if not os.path.isfile(dep_file):
print('* PythonTeX error')
print(' Cannot find dependency "' + dep + '"')
print(' It belongs to ' + key.replace('#', ':'))
print(' Relative paths to dependencies must be specified from the working directory.')
temp_data['errors'] += 1
# A removed dependency should trigger an error, but it
# shouldn't cause code to execute. Running the code
# again would just give more errors when it can't find
# the dependency. (There won't be issues when a
# dependency is added or removed, because that would
# involve modifying code, which would trigger
# re-execution.)
elif hashdependencies:
# Read and hash the file in binary. Opening in text mode
# would require an unnecessary decoding and encoding cycle.
f = open(dep_file, 'rb')
hasher = sha1()
h = hasher(f.read()).hexdigest()
f.close()
if h != old_dep_hash_dict[dep][1]:
return True
else:
mtime = os.path.getmtime(dep_file)
if mtime != old_dep_hash_dict[dep][0]:
return True
return False
def should_rerun(hash, old_hash, old_exit_status, key, rerun, data, old_data, temp_data):
# #### Need to clean up arg passing here
if rerun == 'never':
if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):
print('* PythonTeX warning')
print(' Session ' + key.replace('#', ':') + ' has rerun=never')
print(' But its code or dependencies have been modified')
temp_data['warnings'] += 1
return False
elif rerun == 'modified':
if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):
return True
else:
return False
elif rerun == 'errors':
if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or
old_exit_status[0] != 0):
return True
else:
return False
elif rerun == 'warnings':
if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or
old_exit_status != (0, 0)):
return True
else:
return False
elif rerun == 'always':
return True
def hash_all(data, temp_data, old_data, engine_dict):
'''
Hash the code to see what has changed and needs to be updated.
Save the hashes in hashdict. Create update_code, a list of bools
regarding whether code should be executed. Create update_pygments, a
list of bools determining what needs updated Pygments highlighting.
Update pygments_settings to account for Pygments (as opposed to PythonTeX)
commands and environments.
'''
# Note that the PythonTeX information that accompanies code must be
# hashed in addition to the code itself; the code could stay the same,
# but its context or args could change, which might require that code be
# executed. All of the PythonTeX information is hashed except for the
# input line number. Context-dependent code is going too far if
# it depends on that.
# Create variables to more easily access parts of data
pytxcode = temp_data['pytxcode']
encoding = data['encoding']
loaded_old_data = temp_data['loaded_old_data']
rerun = temp_data['rerun']
pygments_settings = data['pygments_settings']
# Calculate cumulative hashes for all code that is executed
# Calculate individual hashes for all code that will be typeset
code_hasher = defaultdict(sha1)
cons_hasher = defaultdict(sha1)
cc_hasher = defaultdict(sha1)
typeset_hasher = defaultdict(sha1)
for c in pytxcode:
if c.is_code:
code_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))
code_encoded = c.code.encode(encoding)
code_hasher[c.key_run].update(code_encoded)
if c.is_typeset:
typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
typeset_hasher[c.key_typeset].update(code_encoded)
typeset_hasher[c.key_typeset].update(c.args_prettyprint.encode(encoding))
elif c.is_cons:
cons_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))
code_encoded = c.code.encode(encoding)
cons_hasher[c.key_run].update(code_encoded)
if c.is_typeset:
typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
typeset_hasher[c.key_typeset].update(code_encoded)
typeset_hasher[c.key_typeset].update(c.args_prettyprint.encode(encoding))
elif c.is_cc:
cc_hasher[c.cc_type].update(c.hashable_delims_run.encode(encoding))
cc_hasher[c.cc_type].update(c.code.encode(encoding))
elif c.is_typeset:
typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
typeset_hasher[c.key_typeset].update(c.code.encode(encoding))
typeset_hasher[c.key_typeset].update(c.args_prettyprint.encode(encoding))
# Store hashes
code_hash_dict = {}
for key in code_hasher:
family = key.split('#', 1)[0]
code_hash_dict[key] = (code_hasher[key].hexdigest(),
cc_hasher[family].hexdigest(),
engine_dict[family].get_hash())
data['code_hash_dict'] = code_hash_dict
cons_hash_dict = {}
for key in cons_hasher:
family = key.split('#', 1)[0]
cons_hash_dict[key] = (cons_hasher[key].hexdigest(),
cc_hasher[family].hexdigest(),
engine_dict[family].get_hash())
data['cons_hash_dict'] = cons_hash_dict
typeset_hash_dict = {}
for key in typeset_hasher:
typeset_hash_dict[key] = typeset_hasher[key].hexdigest()
data['typeset_hash_dict'] = typeset_hash_dict
# See what needs to be updated.
# In the process, copy over macros and files that may be reused.
code_update = {}
cons_update = {}
pygments_update = {}
macros = defaultdict(list)
files = defaultdict(list)
pygments_macros = {}
pygments_files = {}
typeset_cache = {}
dependencies = defaultdict(dict)
exit_status = {}
pygments_settings_changed = {}
if loaded_old_data:
old_macros = old_data['macros']
old_files = old_data['files']
old_pygments_macros = old_data['pygments_macros']
old_pygments_files = old_data['pygments_files']
old_typeset_cache = old_data['typeset_cache']
old_dependencies = old_data['dependencies']
old_exit_status = old_data['exit_status']
old_code_hash_dict = old_data['code_hash_dict']
old_cons_hash_dict = old_data['cons_hash_dict']
old_typeset_hash_dict = old_data['typeset_hash_dict']
old_pygments_settings = old_data['pygments_settings']
for s in pygments_settings:
if (s in old_pygments_settings and
pygments_settings[s] == old_pygments_settings[s]):
pygments_settings_changed[s] = False
else:
pygments_settings_changed[s] = True
# If old data was loaded (and thus is compatible) determine what has
# changed so that only
# modified code may be executed. Otherwise, execute everything.
# We don't have to worry about checking for changes in pyfuture, because
# custom code and default code are hashed. The treatment of keeptemps
# could be made more efficient (if changed to 'none', just delete old temp
# files rather than running everything again), but given that it is
# intended as a debugging aid, that probable isn't worth it.
# We don't have to worry about hashdependencies changing, because if it
# does the hashes won't match (file contents vs. mtime) and thus code will
# be re-executed.
if loaded_old_data and data['code_vitals'] == old_data['code_vitals']:
# Compare the hash values, and set which code needs to be run
for key in code_hash_dict:
if (key in old_code_hash_dict and
not should_rerun(code_hash_dict[key], old_code_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):
code_update[key] = False
macros[key] = old_macros[key]
files[key] = old_files[key]
dependencies[key] = old_dependencies[key]
exit_status[key] = old_exit_status[key]
else:
code_update[key] = True
else:
for key in code_hash_dict:
code_update[key] = True
if loaded_old_data and data['cons_vitals'] == old_data['cons_vitals']:
# Compare the hash values, and set which code needs to be run
for key in cons_hash_dict:
if (key in old_cons_hash_dict and
not should_rerun(cons_hash_dict[key], old_cons_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):
cons_update[key] = False
macros[key] = old_macros[key]
files[key] = old_files[key]
typeset_cache[key] = old_typeset_cache[key]
dependencies[key] = old_dependencies[key]
exit_status[key] = old_exit_status[key]
else:
cons_update[key] = True
else:
for key in cons_hash_dict:
cons_update[key] = True
if loaded_old_data and data['typeset_vitals'] == old_data['typeset_vitals']:
for key in typeset_hash_dict:
family = key.split('#', 1)[0]
if family in pygments_settings:
if (not pygments_settings_changed[family] and
key in old_typeset_hash_dict and
typeset_hash_dict[key] == old_typeset_hash_dict[key] and
not modified_dependencies(key, data, old_data, temp_data)):
pygments_update[key] = False
if key in old_pygments_macros:
pygments_macros[key] = old_pygments_macros[key]
if key in old_pygments_files:
pygments_files[key] = old_pygments_files[key]
if key in old_dependencies:
dependencies[key] = old_dependencies[key]
else:
pygments_update[key] = True
else:
pygments_update[key] = False
# Make sure Pygments styles are up-to-date
pygments_style_list = list(get_all_styles())
if pygments_style_list != old_data['pygments_style_list']:
pygments_style_defs = {}
# Lazy import
from pygments.formatters import LatexFormatter
for s in pygments_style_list:
formatter = LatexFormatter(style=s, commandprefix='PYG'+s)
pygments_style_defs[s] = formatter.get_style_defs()
else:
pygments_style_defs = old_data['pygments_style_defs']
else:
for key in typeset_hash_dict:
family = key.split('#', 1)[0]
if family in pygments_settings:
pygments_update[key] = True
else:
pygments_update[key] = False
# Create Pygments styles
pygments_style_list = list(get_all_styles())
pygments_style_defs = {}
# Lazy import
from pygments.formatters import LatexFormatter
for s in pygments_style_list:
formatter = LatexFormatter(style=s, commandprefix='PYG'+s)
pygments_style_defs[s] = formatter.get_style_defs()
# Save to data
temp_data['code_update'] = code_update
temp_data['cons_update'] = cons_update
temp_data['pygments_update'] = pygments_update
data['macros'] = macros
data['files'] = files
data['pygments_macros'] = pygments_macros
data['pygments_style_list'] = pygments_style_list
data['pygments_style_defs'] = pygments_style_defs
data['pygments_files'] = pygments_files
data['typeset_cache'] = typeset_cache
data['dependencies'] = dependencies
data['exit_status'] = exit_status
# Clean up for code that will be run again, and for code that no longer
# exists.
if loaded_old_data:
# Take care of code files
for key in code_hash_dict:
if code_update[key] and key in old_files:
for f in old_files[key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
for key in old_code_hash_dict:
if key not in code_hash_dict:
for f in old_files[key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
# Take care of old console files
for key in cons_hash_dict:
if cons_update[key] and key in old_files:
for f in old_files[key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
for key in old_cons_hash_dict:
if key not in cons_hash_dict:
for f in old_files[key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
# Take care of old Pygments files
# The approach here is a little different since there isn't a
# Pygments-specific hash dict, but there is a Pygments-specific
# dict of lists of files.
for key in pygments_update:
if pygments_update[key] and key in old_pygments_files:
for f in old_pygments_files[key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
for key in old_pygments_files:
if key not in pygments_update:
for f in old_pygments_files[key]:
f = os.path.expanduser(os.path.normcase(f))
if os.path.isfile(f):
os.remove(f)
def parse_code_write_scripts(data, temp_data, engine_dict):
'''
Parse the code file into separate scripts, and write them to file.
'''
code_dict = defaultdict(list)
cc_dict_begin = defaultdict(list)
cc_dict_end = defaultdict(list)
cons_dict = defaultdict(list)
pygments_list = []
# Create variables to ease data access
encoding = data['encoding']
utilspath = data['utilspath']
outputdir = data['settings']['outputdir']
workingdir = data['settings']['workingdir']
pytxcode = temp_data['pytxcode']
code_update = temp_data['code_update']
cons_update = temp_data['cons_update']
pygments_update = temp_data['pygments_update']
files = data['files']
debug = temp_data['debug']
interactive = temp_data['interactive']
# Tweak the update dicts to work with debug command-line option.
# #### This should probably be refactored later, once the debug interface