| Home | Trees | Indices | Help |
|
|---|
|
|
1 # -*- coding: utf-8 -*-
2
3
4 __doc__ = """GNUmed general tools."""
5
6 #===========================================================================
7 __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>"
8 __license__ = "GPL v2 or later (details at http://www.gnu.org)"
9
10 # std libs
11 import sys
12 import os
13 import os.path
14 import csv
15 import tempfile
16 import logging
17 import hashlib
18 import platform
19 import subprocess
20 import decimal
21 import getpass
22 import io
23 import functools
24 import json
25 import shutil
26 import zipfile
27 import datetime as pydt
28 import re as regex
29 import xml.sax.saxutils as xml_tools
30 # old:
31 import pickle, zlib
32 # docutils
33 du_core = None
34
35
36 # GNUmed libs
37 if __name__ == '__main__':
38 sys.path.insert(0, '../../')
39 from Gnumed.pycommon import gmBorg
40
41
42 _log = logging.getLogger('gm.tools')
43
44 # CAPitalization modes:
45 ( CAPS_NONE, # don't touch it
46 CAPS_FIRST, # CAP first char, leave rest as is
47 CAPS_ALLCAPS, # CAP all chars
48 CAPS_WORDS, # CAP first char of every word
49 CAPS_NAMES, # CAP in a way suitable for names (tries to be smart)
50 CAPS_FIRST_ONLY # CAP first char, lowercase the rest
51 ) = range(6)
52
53
54 u_currency_pound = '\u00A3' # Pound sign
55 u_currency_sign = '\u00A4' # generic currency sign
56 u_currency_yen = '\u00A5' # Yen sign
57 u_right_double_angle_quote = '\u00AB' # <<
58 u_registered_trademark = '\u00AE'
59 u_plus_minus = '\u00B1'
60 u_superscript_one = '\u00B9' # ^1
61 u_left_double_angle_quote = '\u00BB' # >>
62 u_one_quarter = '\u00BC'
63 u_one_half = '\u00BD'
64 u_three_quarters = '\u00BE'
65 u_multiply = '\u00D7' # x
66 u_greek_ALPHA = '\u0391'
67 u_greek_alpha = '\u03b1'
68 u_greek_OMEGA = '\u03A9'
69 u_greek_omega = '\u03c9'
70 u_dagger = '\u2020'
71 u_triangular_bullet = '\u2023' # triangular bullet (>)
72 u_ellipsis = '\u2026' # ...
73 u_euro = '\u20AC' # EURO sign
74 u_numero = '\u2116' # No. / # sign
75 u_down_left_arrow = '\u21B5' # <-'
76 u_left_arrow = '\u2190' # <--
77 u_up_arrow = '\u2191'
78 u_arrow2right = '\u2192' # -->
79 u_down_arrow = '\u2193'
80 u_left_arrow_with_tail = '\u21a2' # <--<
81 u_arrow2right_from_bar = '\u21a6' # |->
82 u_arrow2right_until_vertical_bar = '\u21e5' # -->|
83 u_sum = '\u2211' # sigma
84 u_almost_equal_to = '\u2248' # approximately / nearly / roughly
85 u_corresponds_to = '\u2258'
86 u_infinity = '\u221E'
87 u_arrow2right_until_vertical_bar2 = '\u2b72' # -->|
88 u_diameter = '\u2300'
89 u_checkmark_crossed_out = '\u237B'
90 u_box_vert_left = '\u23b8'
91 u_box_vert_right = '\u23b9'
92 u_box_horiz_single = '\u2500' # -
93 u_box_vert_light = '\u2502'
94 u_box_horiz_light_3dashes = '\u2504' # ...
95 u_box_vert_light_4dashes = '\u2506'
96 u_box_horiz_4dashes = '\u2508' # ....
97 u_box_T_right = '\u251c' # |-
98 u_box_T_left = '\u2524' # -|
99 u_box_T_down = '\u252c'
100 u_box_T_up = '\u2534'
101 u_box_plus = '\u253c'
102 u_box_top_double = '\u2550'
103 u_box_top_left_double_single = '\u2552'
104 u_box_top_right_double_single = '\u2555'
105 u_box_top_left_arc = '\u256d'
106 u_box_top_right_arc = '\u256e'
107 u_box_bottom_right_arc = '\u256f'
108 u_box_bottom_left_arc = '\u2570'
109 u_box_horiz_light_heavy = '\u257c'
110 u_box_horiz_heavy_light = '\u257e'
111 u_skull_and_crossbones = '\u2620'
112 u_caduceus = '\u2624'
113 u_frowning_face = '\u2639'
114 u_smiling_face = '\u263a'
115 u_black_heart = '\u2665'
116 u_female = '\u2640'
117 u_male = '\u2642'
118 u_male_female = '\u26a5'
119 u_checkmark_thin = '\u2713'
120 u_checkmark_thick = '\u2714'
121 u_heavy_greek_cross = '\u271a'
122 u_arrow2right_thick = '\u2794'
123 u_writing_hand = '\u270d'
124 u_pencil_1 = '\u270e'
125 u_pencil_2 = '\u270f'
126 u_pencil_3 = '\u2710'
127 u_latin_cross = '\u271d'
128 u_arrow2right_until_black_diamond = '\u291e' # ->*
129 u_kanji_yen = '\u5186' # Yen kanji
130 u_replacement_character = '\ufffd'
131 u_link_symbol = '\u1f517'
132
133 _kB = 1024
134 _MB = 1024 * _kB
135 _GB = 1024 * _MB
136 _TB = 1024 * _GB
137 _PB = 1024 * _TB
138
139 #===========================================================================
141
142 print(".========================================================")
143 print("| Unhandled exception caught !")
144 print("| Type :", t)
145 print("| Value:", v)
146 print("`========================================================")
147 _log.critical('unhandled exception caught', exc_info = (t,v,tb))
148 sys.__excepthook__(t,v,tb)
149
150 #===========================================================================
151 # path level operations
152 #---------------------------------------------------------------------------
154 try:
155 if mode is None:
156 os.makedirs(directory)
157 else:
158 old_umask = os.umask(0)
159 os.makedirs(directory, mode)
160 os.umask(old_umask)
161 except OSError as e:
162 if (e.errno == 17) and not os.path.isdir(directory):
163 raise
164 return True
165
166 #---------------------------------------------------------------------------
168 #-------------------------------
169 def _on_rm_error(func, path, exc):
170 _log.error('error while shutil.rmtree(%s)', path, exc_info=exc)
171 return True
172 #-------------------------------
173 error_count = 0
174 try:
175 shutil.rmtree(directory, False, _on_rm_error)
176 except Exception:
177 _log.exception('cannot shutil.rmtree(%s)', directory)
178 error_count += 1
179 return error_count
180
181 #---------------------------------------------------------------------------
183 _log.debug('cleaning out [%s]', directory)
184 try:
185 items = os.listdir(directory)
186 except OSError:
187 return False
188 for item in items:
189 # attempt file/link removal and ignore (but log) errors
190 full_item = os.path.join(directory, item)
191 try:
192 os.remove(full_item)
193 except OSError: # as per the docs, this is a directory
194 _log.debug('[%s] seems to be a subdirectory', full_item)
195 errors = rmdir(full_item)
196 if errors > 0:
197 return False
198 except Exception:
199 _log.exception('cannot os.remove(%s) [a file or a link]', full_item)
200 return False
201
202 return True
203
204 #---------------------------------------------------------------------------
206 if base_dir is None:
207 base_dir = gmPaths().tmp_dir
208 if prefix is None:
209 prefix = 'sandbox-'
210 return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
211
212 #---------------------------------------------------------------------------
215
216 #---------------------------------------------------------------------------
218 # /home/user/dir/ -> dir
219 # /home/user/dir -> dir
220 return os.path.basename(os.path.normpath(directory)) # normpath removes trailing slashes if any
221
222 #---------------------------------------------------------------------------
224 try:
225 return len(os.listdir(directory)) == 0
226 except OSError as exc:
227 if exc.errno == 2:
228 return None
229 raise
230
231 #---------------------------------------------------------------------------
233 """This class provides the following paths:
234
235 .home_dir user home
236 .local_base_dir script installation dir
237 .working_dir current dir
238 .user_config_dir
239 .system_config_dir
240 .system_app_data_dir (not writable)
241 .tmp_dir instance-local
242 .user_tmp_dir user-local (NOT per instance)
243 .bytea_cache_dir caches downloaded BYTEA data
244 """
246 """Setup pathes.
247
248 <app_name> will default to (name of the script - .py)
249 """
250 try:
251 self.already_inited
252 return
253 except AttributeError:
254 pass
255
256 self.init_paths(app_name=app_name, wx=wx)
257 self.already_inited = True
258
259 #--------------------------------------
260 # public API
261 #--------------------------------------
263
264 if wx is None:
265 _log.debug('wxPython not available')
266 _log.debug('detecting paths directly')
267
268 if app_name is None:
269 app_name, ext = os.path.splitext(os.path.basename(sys.argv[0]))
270 _log.info('app name detected as [%s]', app_name)
271 else:
272 _log.info('app name passed in as [%s]', app_name)
273
274 # the user home, doesn't work in Wine so work around that
275 self.__home_dir = None
276
277 # where the main script (the "binary") is installed
278 if getattr(sys, 'frozen', False):
279 _log.info('frozen app, installed into temporary path')
280 # this would find the path of *THIS* file
281 #self.local_base_dir = os.path.dirname(__file__)
282 # while this is documented on the web, the ${_MEIPASS2} does not exist
283 #self.local_base_dir = os.environ.get('_MEIPASS2')
284 # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use
285 # when asking about this on pyinstaller@googlegroups.com
286 #self.local_base_dir = sys._MEIPASS
287 # however, we are --onedir, so we should look at sys.executable
288 # as per the pyinstaller manual
289 self.local_base_dir = os.path.dirname(sys.executable)
290 else:
291 self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
292
293 # the current working dir at the OS
294 self.working_dir = os.path.abspath(os.curdir)
295
296 # user-specific config dir, usually below the home dir
297 mkdir(os.path.join(self.home_dir, '.%s' % app_name))
298 self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name)
299
300 # system-wide config dir, usually below /etc/ under UN*X
301 try:
302 self.system_config_dir = os.path.join('/etc', app_name)
303 except ValueError:
304 #self.system_config_dir = self.local_base_dir
305 self.system_config_dir = self.user_config_dir
306
307 # system-wide application data dir
308 try:
309 self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name)
310 except ValueError:
311 self.system_app_data_dir = self.local_base_dir
312
313 # temporary directory
314 try:
315 self.__tmp_dir_already_set
316 _log.debug('temp dir already set')
317 except AttributeError:
318 _log.info('temp file prefix: %s', tempfile.gettempprefix())
319 _log.info('initial (user level) temp dir: %s', tempfile.gettempdir())
320 # $TMP/gnumed-$USER/
321 self.user_tmp_dir = os.path.join(tempfile.gettempdir(), app_name + '-' + getpass.getuser())
322 mkdir(self.user_tmp_dir, 0o700)
323 tempfile.tempdir = self.user_tmp_dir
324 _log.info('intermediate (app level) temp dir: %s', tempfile.gettempdir())
325 # $TMP/gnumed-$USER/g$UNIQUE/
326 self.tmp_dir = tempfile.mkdtemp(prefix = 'g-')
327 _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir())
328
329 # BYTEA cache dir
330 cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache')
331 try:
332 stat = os.stat(cache_dir)
333 _log.warning('reusing BYTEA cache dir: %s', cache_dir)
334 _log.debug(stat)
335 except FileNotFoundError:
336 mkdir(cache_dir, mode = 0o0700)
337 self.bytea_cache_dir = cache_dir
338
339 self.__log_paths()
340 if wx is None:
341 return True
342
343 # retry with wxPython
344 _log.debug('re-detecting paths with wxPython')
345
346 std_paths = wx.StandardPaths.Get()
347 _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName())
348
349 # user-specific config dir, usually below the home dir
350 mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name))
351 self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)
352
353 # system-wide config dir, usually below /etc/ under UN*X
354 try:
355 tmp = std_paths.GetConfigDir()
356 if not tmp.endswith(app_name):
357 tmp = os.path.join(tmp, app_name)
358 self.system_config_dir = tmp
359 except ValueError:
360 # leave it at what it was from direct detection
361 pass
362
363 # system-wide application data dir
364 # Robin attests that the following doesn't always
365 # give sane values on Windows, so IFDEF it
366 if 'wxMSW' in wx.PlatformInfo:
367 _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir')
368 else:
369 try:
370 self.system_app_data_dir = std_paths.GetDataDir()
371 except ValueError:
372 pass
373
374 self.__log_paths()
375 return True
376
377 #--------------------------------------
379 _log.debug('sys.argv[0]: %s', sys.argv[0])
380 _log.debug('sys.executable: %s', sys.executable)
381 _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>'))
382 _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>'))
383 _log.debug('__file__ : %s', __file__)
384 _log.debug('local application base dir: %s', self.local_base_dir)
385 _log.debug('current working dir: %s', self.working_dir)
386 _log.debug('user home dir: %s', self.home_dir)
387 _log.debug('user-specific config dir: %s', self.user_config_dir)
388 _log.debug('system-wide config dir: %s', self.system_config_dir)
389 _log.debug('system-wide application data dir: %s', self.system_app_data_dir)
390 _log.debug('temporary dir (user): %s', self.user_tmp_dir)
391 _log.debug('temporary dir (instance): %s', self.tmp_dir)
392 _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
393
394 #--------------------------------------
395 # properties
396 #--------------------------------------
398 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
399 msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
400 _log.error(msg)
401 raise ValueError(msg)
402 self.__user_config_dir = path
403
406
407 user_config_dir = property(_get_user_config_dir, _set_user_config_dir)
408 #--------------------------------------
410 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
411 msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
412 _log.error(msg)
413 raise ValueError(msg)
414 self.__system_config_dir = path
415
418
419 system_config_dir = property(_get_system_config_dir, _set_system_config_dir)
420 #--------------------------------------
422 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
423 msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path)
424 _log.error(msg)
425 raise ValueError(msg)
426 self.__system_app_data_dir = path
427
430
431 system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir)
432 #--------------------------------------
435
437 if self.__home_dir is not None:
438 return self.__home_dir
439
440 tmp = os.path.expanduser('~')
441 if tmp == '~':
442 _log.error('this platform does not expand ~ properly')
443 try:
444 tmp = os.environ['USERPROFILE']
445 except KeyError:
446 _log.error('cannot access $USERPROFILE in environment')
447
448 if not (
449 os.access(tmp, os.R_OK)
450 and
451 os.access(tmp, os.X_OK)
452 and
453 os.access(tmp, os.W_OK)
454 ):
455 msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp)
456 _log.error(msg)
457 raise ValueError(msg)
458
459 self.__home_dir = tmp
460 return self.__home_dir
461
462 home_dir = property(_get_home_dir, _set_home_dir)
463
464 #--------------------------------------
466 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
467 msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path)
468 _log.error(msg)
469 raise ValueError(msg)
470 _log.debug('previous temp dir: %s', tempfile.gettempdir())
471 self.__tmp_dir = path
472 tempfile.tempdir = self.__tmp_dir
473 _log.debug('new temp dir: %s', tempfile.gettempdir())
474 self.__tmp_dir_already_set = True
475
478
479 tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
480
481 #===========================================================================
482 # file related tools
483 #---------------------------------------------------------------------------
484 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
485 if target_encoding is None:
486 return source_file
487 if target_encoding == source_encoding:
488 return source_file
489 if target_file is None:
490 target_file = get_unique_filename (
491 prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding),
492 suffix = fname_extension(source_file, '.txt'),
493 tmp_dir = base_dir
494 )
495
496 _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file)
497
498 in_file = io.open(source_file, mode = 'rt', encoding = source_encoding)
499 out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode)
500 for line in in_file:
501 out_file.write(line)
502 out_file.close()
503 in_file.close()
504
505 return target_file
506
507 #---------------------------------------------------------------------------
509 _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir)
510 success = False
511 try:
512 with zipfile.ZipFile(archive_name) as archive:
513 archive.extractall(target_dir)
514 success = True
515 except Exception:
516 _log.exception('cannot unzip')
517 return False
518 if remove_archive:
519 remove_file(archive_name)
520 return success
521
522 #---------------------------------------------------------------------------
524 if not os.path.lexists(filename):
525 return True
526
527 # attempt file remove and ignore (but log) errors
528 try:
529 os.remove(filename)
530 return True
531
532 except Exception:
533 if log_error:
534 _log.exception('cannot os.remove(%s)', filename)
535
536 if force:
537 tmp_name = get_unique_filename(tmp_dir = fname_dir(filename))
538 _log.debug('attempting os.replace() to: %s', tmp_name)
539 try:
540 os.replace(filename, tmp_name)
541 return True
542
543 except BaseException:
544 if log_error:
545 _log.exception('cannot os.remove(%s)', filename)
546
547 return False
548
549 #---------------------------------------------------------------------------
551 blocksize = 2**10 * 128 # 128k, since md5 uses 128 byte blocks
552 _log.debug('md5(%s): <%s> byte blocks', filename, blocksize)
553
554 f = io.open(filename, mode = 'rb')
555
556 md5 = hashlib.md5()
557 while True:
558 data = f.read(blocksize)
559 if not data:
560 break
561 md5.update(data)
562 f.close()
563
564 _log.debug('md5(%s): %s', filename, md5.hexdigest())
565
566 if return_hex:
567 return md5.hexdigest()
568 return md5.digest()
569
570 #---------------------------------------------------------------------------
572 _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size)
573 md5_concat = ''
574 f = open(filename, 'rb')
575 while True:
576 md5 = hashlib.md5()
577 data = f.read(chunk_size)
578 if not data:
579 break
580 md5.update(data)
581 md5_concat += md5.hexdigest()
582 f.close()
583 md5 = hashlib.md5()
584 md5.update(md5_concat)
585 hex_digest = md5.hexdigest()
586 _log.debug('md5("%s"): %s', md5_concat, hex_digest)
587 return hex_digest
588
589 #---------------------------------------------------------------------------
590 default_csv_reader_rest_key = 'list_of_values_of_unknown_fields'
591
593 try:
594 is_dict_reader = kwargs['dict']
595 del kwargs['dict']
596 except KeyError:
597 is_dict_reader = False
598
599 if is_dict_reader:
600 kwargs['restkey'] = default_csv_reader_rest_key
601 return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs)
602 return csv.reader(csv_data, dialect=dialect, **kwargs)
603
604
605
606
610
611 #def utf_8_encoder(unicode_csv_data):
612 # for line in unicode_csv_data:
613 # yield line.encode('utf-8')
614
616
617 # csv.py doesn't do Unicode; encode temporarily as UTF-8:
618 try:
619 is_dict_reader = kwargs['dict']
620 del kwargs['dict']
621 if is_dict_reader is not True:
622 raise KeyError
623 kwargs['restkey'] = default_csv_reader_rest_key
624 csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
625 except KeyError:
626 is_dict_reader = False
627 csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
628
629 for row in csv_reader:
630 # decode ENCODING back to Unicode, cell by cell:
631 if is_dict_reader:
632 for key in row.keys():
633 if key == default_csv_reader_rest_key:
634 old_data = row[key]
635 new_data = []
636 for val in old_data:
637 new_data.append(str(val, encoding))
638 row[key] = new_data
639 if default_csv_reader_rest_key not in csv_reader.fieldnames:
640 csv_reader.fieldnames.append(default_csv_reader_rest_key)
641 else:
642 row[key] = str(row[key], encoding)
643 yield row
644 else:
645 yield [ str(cell, encoding) for cell in row ]
646 #yield [str(cell, 'utf-8') for cell in row]
647
648 #---------------------------------------------------------------------------
650 """Normalizes unicode, removes non-alpha characters, converts spaces to underscores."""
651
652 dir_part, name_part = os.path.split(filename)
653 if name_part == '':
654 return filename
655
656 import unicodedata
657 name_part = unicodedata.normalize('NFKD', name_part)
658 # remove everything not in group []
659 name_part = regex.sub (
660 '[^.\w\s[\]()%§+-]',
661 '',
662 name_part,
663 flags = regex.UNICODE
664 ).strip()
665 # translate whitespace to underscore
666 name_part = regex.sub (
667 '\s+',
668 '_',
669 name_part,
670 flags = regex.UNICODE
671 )
672 return os.path.join(dir_part, name_part)
673
674 #---------------------------------------------------------------------------
676 """/home/user/dir/filename.ext -> filename"""
677 return os.path.splitext(os.path.basename(filename))[0]
678
679 #---------------------------------------------------------------------------
681 """/home/user/dir/filename.ext -> /home/user/dir/filename"""
682 return os.path.splitext(filename)[0]
683
684 #---------------------------------------------------------------------------
686 """ /home/user/dir/filename.ext -> .ext
687 '' or '.' -> fallback if any else ''
688 """
689 ext = os.path.splitext(filename)[1]
690 if ext.strip() not in ['.', '']:
691 return ext
692 if fallback is None:
693 return ''
694 return fallback
695
696 #---------------------------------------------------------------------------
700
701 #---------------------------------------------------------------------------
705
706 #---------------------------------------------------------------------------
708 """This function has a race condition between
709 its file.close()
710 and actually
711 using the filename in callers.
712
713 The file will NOT exist after calling this function.
714 """
715 if tmp_dir is not None:
716 if (
717 not os.access(tmp_dir, os.F_OK)
718 or
719 not os.access(tmp_dir, os.X_OK | os.W_OK)
720 ):
721 _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir)
722 tmp_dir = None
723
724 if include_timestamp:
725 ts = pydt.datetime.now().strftime('%m%d-%H%M%S-')
726 else:
727 ts = ''
728
729 kwargs = {
730 'dir': tmp_dir,
731 # make sure file gets deleted as soon as
732 # .close()d so we can "safely" open it again
733 'delete': True
734 }
735
736 if prefix is None:
737 kwargs['prefix'] = 'gm-%s' % ts
738 else:
739 kwargs['prefix'] = prefix + ts
740
741 if suffix in [None, '']:
742 kwargs['suffix'] = '.tmp'
743 else:
744 if not suffix.startswith('.'):
745 suffix = '.' + suffix
746 kwargs['suffix'] = suffix
747
748 f = tempfile.NamedTemporaryFile(**kwargs)
749 filename = f.name
750 f.close()
751
752 return filename
753
754 #---------------------------------------------------------------------------
756 import ctypes
757 #windows_create_symlink = ctypes.windll.kernel32.CreateSymbolicLinkW
758 kernel32 = ctype.WinDLL('kernel32', use_last_error = True)
759 windows_create_symlink = kernel32.CreateSymbolicLinkW
760 windows_create_symlink.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
761 windows_create_symlink.restype = ctypes.c_ubyte
762 if os.path.isdir(physical_name):
763 flags = 1
764 else:
765 flags = 0
766 ret_code = windows_create_symlink(link_name, physical_name.replace('/', '\\'), flags)
767 _log.debug('ctypes.windll.kernel32.CreateSymbolicLinkW() [%s] exit code: %s', windows_create_symlink, ret_code)
768 if ret_code == 0:
769 raise ctypes.WinError()
770 return ret_code
771
772 #---------------------------------------------------------------------------
774
775 _log.debug('creating symlink (overwrite = %s):', overwrite)
776 _log.debug('link [%s] =>', link_name)
777 _log.debug('=> physical [%s]', physical_name)
778
779 if os.path.exists(link_name):
780 _log.debug('link exists')
781 if overwrite:
782 return True
783 return False
784
785 try:
786 os.symlink(physical_name, link_name)
787 except (AttributeError, NotImplementedError):
788 _log.debug('this Python does not have os.symlink(), trying via ctypes')
789 __make_symlink_on_windows(physical_name, link_name)
790 except PermissionError:
791 _log.exception('cannot create link')
792 return False
793 #except OSError:
794 # unpriviledged on Windows
795 return True
796
797 #===========================================================================
798 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
799 """Import a module from any location."""
800
801 _log.debug('CWD: %s', os.getcwd())
802
803 remove_path = always_remove_path or False
804 if module_path not in sys.path:
805 _log.info('appending to sys.path: [%s]' % module_path)
806 sys.path.append(module_path)
807 remove_path = True
808
809 _log.debug('will remove import path: %s', remove_path)
810
811 if module_name.endswith('.py'):
812 module_name = module_name[:-3]
813
814 try:
815 module = __import__(module_name)
816 except Exception:
817 _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path))
818 while module_path in sys.path:
819 sys.path.remove(module_path)
820 raise
821
822 _log.info('imported module [%s] as [%s]' % (module_name, module))
823 if remove_path:
824 while module_path in sys.path:
825 sys.path.remove(module_path)
826
827 return module
828
829 #===========================================================================
830 # text related tools
831 #---------------------------------------------------------------------------
833 if size == 1:
834 return template % _('1 Byte')
835 if size < 10 * _kB:
836 return template % _('%s Bytes') % size
837 if size < _MB:
838 return template % '%.1f kB' % (float(size) / _kB)
839 if size < _GB:
840 return template % '%.1f MB' % (float(size) / _MB)
841 if size < _TB:
842 return template % '%.1f GB' % (float(size) / _GB)
843 if size < _PB:
844 return template % '%.1f TB' % (float(size) / _TB)
845 return template % '%.1f PB' % (float(size) / _PB)
846
847 #---------------------------------------------------------------------------
849 if boolean is None:
850 return none_return
851 if boolean:
852 return true_return
853 if not boolean:
854 return false_return
855 raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
856
857 #---------------------------------------------------------------------------
859 return bool2subst (
860 boolean = bool(boolean),
861 true_return = true_str,
862 false_return = false_str
863 )
864
865 #---------------------------------------------------------------------------
867 """Modelled after the SQL NULLIF function."""
868 if value is None:
869 return None
870 if strip_string:
871 stripped = value.strip()
872 else:
873 stripped = value
874 if stripped == none_equivalent:
875 return None
876 return value
877
878 #---------------------------------------------------------------------------
879 -def coalesce(initial=None, instead=None, template_initial=None, template_instead=None, none_equivalents=None, function_initial=None):
880 """Modelled after the SQL coalesce function.
881
882 To be used to simplify constructs like:
883
884 if initial is None (or in none_equivalents):
885 real_value = (template_instead % instead) or instead
886 else:
887 real_value = (template_initial % initial) or initial
888 print real_value
889
890 @param initial: the value to be tested for <None>
891 @type initial: any Python type, must have a __str__ method if template_initial is not None
892 @param instead: the value to be returned if <initial> is None
893 @type instead: any Python type, must have a __str__ method if template_instead is not None
894 @param template_initial: if <initial> is returned replace the value into this template, must contain one <%s>
895 @type template_initial: string or None
896 @param template_instead: if <instead> is returned replace the value into this template, must contain one <%s>
897 @type template_instead: string or None
898
899 example:
900 function_initial = ('strftime', '%Y-%m-%d')
901
902 Ideas:
903 - list of insteads: initial, [instead, template], [instead, template], [instead, template], template_initial, ...
904 """
905 if none_equivalents is None:
906 none_equivalents = [None]
907
908 if initial in none_equivalents:
909
910 if template_instead is None:
911 return instead
912
913 return template_instead % instead
914
915 if function_initial is not None:
916 funcname, args = function_initial
917 func = getattr(initial, funcname)
918 initial = func(args)
919
920 if template_initial is None:
921 return initial
922
923 try:
924 return template_initial % initial
925 except TypeError:
926 return template_initial
927
928 #---------------------------------------------------------------------------
930 val = match_obj.group(0).lower()
931 if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']: # FIXME: this needs to expand, configurable ?
932 return val
933 buf = list(val)
934 buf[0] = buf[0].upper()
935 for part in ['mac', 'mc', 'de', 'la']:
936 if len(val) > len(part) and val[:len(part)] == part:
937 buf[len(part)] = buf[len(part)].upper()
938 return ''.join(buf)
939
940 #---------------------------------------------------------------------------
942 """Capitalize the first character but leave the rest alone.
943
944 Note that we must be careful about the locale, this may
945 have issues ! However, for UTF strings it should just work.
946 """
947 if (mode is None) or (mode == CAPS_NONE):
948 return text
949
950 if len(text) == 0:
951 return text
952
953 if mode == CAPS_FIRST:
954 if len(text) == 1:
955 return text[0].upper()
956 return text[0].upper() + text[1:]
957
958 if mode == CAPS_ALLCAPS:
959 return text.upper()
960
961 if mode == CAPS_FIRST_ONLY:
962 # if len(text) == 1:
963 # return text[0].upper()
964 return text[0].upper() + text[1:].lower()
965
966 if mode == CAPS_WORDS:
967 #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
968 return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
969
970 if mode == CAPS_NAMES:
971 #return regex.sub(r'\w+', __cap_name, text)
972 return capitalize(text=text, mode=CAPS_FIRST) # until fixed
973
974 print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode)
975 return text
976
977 #---------------------------------------------------------------------------
979
980 if isinstance(initial, decimal.Decimal):
981 return True, initial
982
983 val = initial
984
985 # float ? -> to string first
986 if type(val) == type(float(1.4)):
987 val = str(val)
988
989 # string ? -> "," to "."
990 if isinstance(val, str):
991 val = val.replace(',', '.', 1)
992 val = val.strip()
993
994 try:
995 d = decimal.Decimal(val)
996 return True, d
997 except (TypeError, decimal.InvalidOperation):
998 return False, val
999
1000 #---------------------------------------------------------------------------
1002
1003 val = initial
1004
1005 # string ? -> "," to "."
1006 if isinstance(val, str):
1007 val = val.replace(',', '.', 1)
1008 val = val.strip()
1009
1010 try:
1011 int_val = int(val)
1012 except (TypeError, ValueError):
1013 _log.exception('int(%s) failed', val)
1014 return False, initial
1015
1016 if minval is not None:
1017 if int_val < minval:
1018 _log.debug('%s < min (%s)', val, minval)
1019 return False, initial
1020 if maxval is not None:
1021 if int_val > maxval:
1022 _log.debug('%s > max (%s)', val, maxval)
1023 return False, initial
1024
1025 return True, int_val
1026
1027 #---------------------------------------------------------------------------
1029 if remove_repeats:
1030 if remove_whitespace:
1031 while text.lstrip().startswith(prefix):
1032 text = text.lstrip().replace(prefix, '', 1).lstrip()
1033 return text
1034 while text.startswith(prefix):
1035 text = text.replace(prefix, '', 1)
1036 return text
1037 if remove_whitespace:
1038 return text.lstrip().replace(prefix, '', 1).lstrip()
1039 return text.replace(prefix, '', 1)
1040
1041 #---------------------------------------------------------------------------
1043 suffix_len = len(suffix)
1044 if remove_repeats:
1045 if remove_whitespace:
1046 while text.rstrip().endswith(suffix):
1047 text = text.rstrip()[:-suffix_len].rstrip()
1048 return text
1049 while text.endswith(suffix):
1050 text = text[:-suffix_len]
1051 return text
1052 if remove_whitespace:
1053 return text.rstrip()[:-suffix_len].rstrip()
1054 return text[:-suffix_len]
1055
1056 #---------------------------------------------------------------------------
1058 if lines is None:
1059 lines = text.split(eol)
1060
1061 while True:
1062 if lines[0].strip(eol).strip() != '':
1063 break
1064 lines = lines[1:]
1065
1066 if return_list:
1067 return lines
1068
1069 return eol.join(lines)
1070
1071 #---------------------------------------------------------------------------
1073 if lines is None:
1074 lines = text.split(eol)
1075
1076 while True:
1077 if lines[-1].strip(eol).strip() != '':
1078 break
1079 lines = lines[:-1]
1080
1081 if return_list:
1082 return lines
1083
1084 return eol.join(lines)
1085
1086 #---------------------------------------------------------------------------
1088 return strip_trailing_empty_lines (
1089 lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True),
1090 text = None,
1091 eol = eol,
1092 return_list = return_list
1093 )
1094
1095 #---------------------------------------------------------------------------
1096 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True):
1097
1098 if len(lines) == 0:
1099 return ''
1100
1101 if strip_leading_empty_lines:
1102 lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True)
1103
1104 if strip_trailing_empty_lines:
1105 lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True)
1106
1107 if strip_trailing_whitespace:
1108 lines = [ l.rstrip() for l in lines ]
1109
1110 indented_lines = [initial_indent + lines[0]]
1111 indented_lines.extend([ subsequent_indent + l for l in lines[1:] ])
1112
1113 return eol.join(indented_lines)
1114
1115 #---------------------------------------------------------------------------
1117 """A word-wrap function that preserves existing line breaks
1118 and most spaces in the text. Expects that existing line
1119 breaks are posix newlines (\n).
1120 """
1121 if width is None:
1122 return text
1123 wrapped = initial_indent + functools.reduce (
1124 lambda line, word, width=width: '%s%s%s' % (
1125 line,
1126 ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)],
1127 word
1128 ),
1129 text.split(' ')
1130 )
1131
1132 if subsequent_indent != '':
1133 wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n'))
1134
1135 if eol != '\n':
1136 wrapped = wrapped.replace('\n', eol)
1137
1138 return wrapped
1139
1140 #---------------------------------------------------------------------------
1141 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
1142
1143 text = text.replace('\r', '')
1144 lines = text.split('\n')
1145 text = ''
1146 for line in lines:
1147
1148 if strip_whitespace:
1149 line = line.strip().strip('\t').strip()
1150
1151 if remove_empty_lines:
1152 if line == '':
1153 continue
1154
1155 text += ('%s%s' % (line, line_separator))
1156
1157 text = text.rstrip(line_separator)
1158
1159 if max_length is not None:
1160 text = text[:max_length]
1161
1162 text = text.rstrip(line_separator)
1163
1164 return text
1165
1166 #---------------------------------------------------------------------------
1168
1169 if len(text) <= max_length:
1170 return text
1171
1172 return text[:max_length-1] + u_ellipsis
1173
1174 #---------------------------------------------------------------------------
1175 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
1176 if text is None:
1177 return None
1178 if max_length is None:
1179 max_length = len(text)
1180 else:
1181 if len(text) <= max_length:
1182 return text
1183 old_words = regex.split('\s+', text, flags = regex.UNICODE)
1184 no_old_words = len(old_words)
1185 max_word_length = max(min_word_length, (max_length // no_old_words))
1186 words = []
1187 for word in old_words:
1188 if len(word) <= max_word_length:
1189 words.append(word)
1190 continue
1191 if ignore_numbers:
1192 tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '')
1193 if tmp.isdigit():
1194 words.append(word)
1195 continue
1196 words.append(word[:max_word_length] + ellipsis)
1197 return ' '.join(words)
1198
1199 #---------------------------------------------------------------------------
1203
1204 #---------------------------------------------------------------------------
1205 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
1206 """Check for special TeX characters and transform them.
1207
1208 replace_eol:
1209 replaces "\n" with "\\newline"
1210 keep_visual_eol:
1211 replaces "\n" with "\\newline \n" such that
1212 both LaTeX will know to place a line break
1213 at this point as well as the visual formatting
1214 is preserved in the LaTeX source (think multi-
1215 row table cells)
1216 """
1217 text = text.replace('\\', '\\textbackslash') # requires \usepackage{textcomp} in LaTeX source
1218 text = text.replace('^', '\\textasciicircum')
1219 text = text.replace('~', '\\textasciitilde')
1220
1221 text = text.replace('{', '\\{')
1222 text = text.replace('}', '\\}')
1223 text = text.replace('%', '\\%')
1224 text = text.replace('&', '\\&')
1225 text = text.replace('#', '\\#')
1226 text = text.replace('$', '\\$')
1227 text = text.replace('_', '\\_')
1228 if replace_eol:
1229 if keep_visual_eol:
1230 text = text.replace('\n', '\\newline \n')
1231 else:
1232 text = text.replace('\n', '\\newline ')
1233
1234 if replace_known_unicode:
1235 # this should NOT be replaced for Xe(La)Tex
1236 text = text.replace(u_euro, '\\EUR') # requires \usepackage{textcomp} in LaTeX source
1237 text = text.replace(u_sum, '$\\Sigma$')
1238
1239 return text
1240
1241 #---------------------------------------------------------------------------
1243 global du_core
1244 if du_core is None:
1245 try:
1246 from docutils import core as du_core
1247 except ImportError:
1248 _log.warning('cannot turn ReST into LaTeX: docutils not installed')
1249 return tex_escape_string(text = rst_text)
1250
1251 parts = du_core.publish_parts (
1252 source = rst_text.replace('\\', '\\\\'),
1253 source_path = '<internal>',
1254 writer_name = 'latex',
1255 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
1256 settings_overrides = {
1257 'input_encoding': 'unicode' # un-encoded unicode
1258 },
1259 enable_exit_status = True # how to use ?
1260 )
1261 return parts['body']
1262
1263 #---------------------------------------------------------------------------
1265 global du_core
1266 if du_core is None:
1267 try:
1268 from docutils import core as du_core
1269 except ImportError:
1270 _log.warning('cannot turn ReST into HTML: docutils not installed')
1271 return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False)
1272
1273 parts = du_core.publish_parts (
1274 source = rst_text.replace('\\', '\\\\'),
1275 source_path = '<internal>',
1276 writer_name = 'latex',
1277 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
1278 settings_overrides = {
1279 'input_encoding': 'unicode' # un-encoded unicode
1280 },
1281 enable_exit_status = True # how to use ?
1282 )
1283 return parts['body']
1284
1285 #---------------------------------------------------------------------------
1287 # a web search did not reveal anything else for Xe(La)Tex
1288 # as opposed to LaTeX, except true unicode chars
1289 return tex_escape_string(text = text, replace_known_unicode = False)
1290
1291 #---------------------------------------------------------------------------
1292 __html_escape_table = {
1293 "&": "&",
1294 '"': """,
1295 "'": "'",
1296 ">": ">",
1297 "<": "<",
1298 }
1299
1301 text = ''.join(__html_escape_table.get(char, char) for char in text)
1302 if replace_eol:
1303 if keep_visual_eol:
1304 text = text.replace('\n', '<br>\n')
1305 else:
1306 text = text.replace('\n', '<br>')
1307 return text
1308
1309 #---------------------------------------------------------------------------
1312
1313 #---------------------------------------------------------------------------
1315 if isinstance(obj, pydt.datetime):
1316 return obj.isoformat()
1317 raise TypeError('cannot json_serialize(%s)' % type(obj))
1318
1319 #---------------------------------------------------------------------------
1320 #---------------------------------------------------------------------------
1322 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2))
1323 try:
1324 d1 = dict(d1)
1325 except TypeError:
1326 pass
1327 try:
1328 d2 = dict(d2)
1329 except TypeError:
1330 pass
1331 keys_d1 = frozenset(d1.keys())
1332 keys_d2 = frozenset(d2.keys())
1333 different = False
1334 if len(keys_d1) != len(keys_d2):
1335 _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2))
1336 different = True
1337 for key in keys_d1:
1338 if key in keys_d2:
1339 if type(d1[key]) != type(d2[key]):
1340 _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key]))
1341 _log.info('%25.25s type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key]))
1342 different = True
1343 continue
1344 if d1[key] == d2[key]:
1345 _log.info('%25.25s: both = >>>%s<<<' % (key, d1[key]))
1346 else:
1347 _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key]))
1348 _log.info('%25.25s dict2 = >>>%s<<<' % ('', d2[key]))
1349 different = True
1350 else:
1351 _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key]))
1352 different = True
1353 for key in keys_d2:
1354 if key in keys_d1:
1355 continue
1356 _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key]))
1357 different = True
1358 if different:
1359 _log.info('dict-likes appear to be different from each other')
1360 return False
1361 _log.info('dict-likes appear equal to each other')
1362 return True
1363
1364 #---------------------------------------------------------------------------
1365 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
1366
1367 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2))
1368 append_type = False
1369 if None not in [title_left, title_right]:
1370 append_type = True
1371 type_left = type(d1)
1372 type_right = type(d2)
1373 if title_left is None:
1374 title_left = '%s' % type_left
1375 if title_right is None:
1376 title_right = '%s' % type_right
1377
1378 try: d1 = dict(d1)
1379 except TypeError: pass
1380 try: d2 = dict(d2)
1381 except TypeError: pass
1382 keys_d1 = d1.keys()
1383 keys_d2 = d2.keys()
1384 data = {}
1385 for key in keys_d1:
1386 data[key] = [d1[key], ' ']
1387 if key in d2:
1388 data[key][1] = d2[key]
1389 for key in keys_d2:
1390 if key in keys_d1:
1391 continue
1392 data[key] = [' ', d2[key]]
1393 max1 = max([ len('%s' % k) for k in keys_d1 ])
1394 max2 = max([ len('%s' % k) for k in keys_d2 ])
1395 max_len = max(max1, max2, len(_('<type>')))
1396 max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's'
1397 max1 = max([ len('%s' % d1[k]) for k in keys_d1 ])
1398 max2 = max([ len('%s' % d2[k]) for k in keys_d2 ])
1399 max_data_len = min(max(max1, max2), 100)
1400 max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's'
1401 diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's'
1402 line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s'
1403
1404 lines = []
1405 # debugging:
1406 #lines.append(u' (40 regular spaces)')
1407 #lines.append((u' ' * 40) + u"(u' ' * 40)")
1408 #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')")
1409 #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')")
1410 #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')")
1411 #lines.append(line_template)
1412 lines.append(line_template % ('', '', title_left, title_right))
1413 if append_type:
1414 lines.append(line_template % ('', _('<type>'), type_left, type_right))
1415
1416 if ignore_diff_in_keys is None:
1417 ignore_diff_in_keys = []
1418
1419 for key in keys_d1:
1420 append_type = False
1421 txt_left_col = '%s' % d1[key]
1422 try:
1423 txt_right_col = '%s' % d2[key]
1424 if type(d1[key]) != type(d2[key]):
1425 append_type = True
1426 except KeyError:
1427 txt_right_col = missing_string
1428 lines.append(line_template % (
1429 bool2subst (
1430 ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)),
1431 '',
1432 difference_indicator
1433 ),
1434 key,
1435 shorten_text(txt_left_col, max_data_len),
1436 shorten_text(txt_right_col, max_data_len)
1437 ))
1438 if append_type:
1439 lines.append(line_template % (
1440 '',
1441 _('<type>'),
1442 shorten_text('%s' % type(d1[key]), max_data_len),
1443 shorten_text('%s' % type(d2[key]), max_data_len)
1444 ))
1445
1446 for key in keys_d2:
1447 if key in keys_d1:
1448 continue
1449 lines.append(line_template % (
1450 bool2subst((key in ignore_diff_in_keys), '', difference_indicator),
1451 key,
1452 shorten_text(missing_string, max_data_len),
1453 shorten_text('%s' % d2[key], max_data_len)
1454 ))
1455
1456 return lines
1457
1458 #---------------------------------------------------------------------------
1459 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
1460 if values2ignore is None:
1461 values2ignore = []
1462 if template is not None:
1463 # all keys in template better exist in d
1464 try:
1465 return template % d
1466 except KeyError:
1467 # or else
1468 _log.exception('template contains %%()s key(s) which do not exist in data dict')
1469 # try to extend dict <d> to contain all required keys,
1470 # for that to work <relevant_keys> better list all
1471 # keys used in <template>
1472 if relevant_keys is not None:
1473 for key in relevant_keys:
1474 try:
1475 d[key]
1476 except KeyError:
1477 d[key] = missing_key_template % {'key': key}
1478 return template % d
1479
1480 if relevant_keys is None:
1481 relevant_keys = list(d.keys())
1482 lines = []
1483 if value_delimiters is None:
1484 delim_left = ''
1485 delim_right = ''
1486 else:
1487 delim_left, delim_right = value_delimiters
1488 if tabular:
1489 max_len = max([ len('%s' % k) for k in relevant_keys ])
1490 max_len_str = '%s.%s' % (max_len, max_len)
1491 line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right))
1492 else:
1493 line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right)
1494 for key in relevant_keys:
1495 try:
1496 val = d[key]
1497 except KeyError:
1498 continue
1499 if val not in values2ignore:
1500 lines.append(line_template % (key, val))
1501 if eol is None:
1502 return lines
1503 return eol.join(lines)
1504
1505 #---------------------------------------------------------------------------
1507 for key in required_keys:
1508 try:
1509 d[key]
1510 except KeyError:
1511 if missing_key_template is None:
1512 d[key] = None
1513 else:
1514 d[key] = missing_key_template % {'key': key}
1515 return d
1516
1517 #---------------------------------------------------------------------------
1518 #---------------------------------------------------------------------------
1520 """Obtains entry from standard input.
1521
1522 prompt: Prompt text to display in standard output
1523 default: Default value (for user to press enter only)
1524 CTRL-C: aborts and returns None
1525 """
1526 if prompt is None:
1527 msg = '(CTRL-C aborts)'
1528 else:
1529 msg = '%s (CTRL-C aborts)' % prompt
1530
1531 if default is None:
1532 msg = msg + ': '
1533 else:
1534 msg = '%s [%s]: ' % (msg, default)
1535
1536 try:
1537 usr_input = input(msg)
1538 except KeyboardInterrupt:
1539 return None
1540
1541 if usr_input == '':
1542 return default
1543
1544 return usr_input
1545
1546 #===========================================================================
1547 # image handling tools
1548 #---------------------------------------------------------------------------
1549 # builtin (ugly but tried and true) fallback icon
1550 __icon_serpent = \
1551 """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\
1552 \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\
1553 \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\
1554 \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\
1555 \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\
1556 \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\
1557 \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec"""
1558
1560
1561 paths = gmPaths(app_name = 'gnumed', wx = wx)
1562
1563 candidates = [
1564 os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
1565 os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
1566 os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'),
1567 os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png')
1568 ]
1569
1570 found_as = None
1571 for candidate in candidates:
1572 try:
1573 open(candidate, 'r').close()
1574 found_as = candidate
1575 break
1576 except IOError:
1577 _log.debug('icon not found in [%s]', candidate)
1578
1579 if found_as is None:
1580 _log.warning('no icon file found, falling back to builtin (ugly) icon')
1581 icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent)))
1582 icon.CopyFromBitmap(icon_bmp_data)
1583 else:
1584 _log.debug('icon found in [%s]', found_as)
1585 icon = wx.Icon()
1586 try:
1587 icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY) #_PNG
1588 except AttributeError:
1589 _log.exception("this platform doesn't support wx.Icon().LoadFile()")
1590
1591 return icon
1592
1593 #---------------------------------------------------------------------------
1595 assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified'
1596
1597 try:
1598 import pyqrcode
1599 except ImportError:
1600 _log.exception('cannot import <pyqrcode>')
1601 return None
1602 if text is None:
1603 with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file:
1604 text = input_file.read()
1605 if qr_filename is None:
1606 if filename is None:
1607 qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png')
1608 else:
1609 qr_filename = get_unique_filename (
1610 prefix = fname_stem(filename) + '-',
1611 suffix = fname_extension(filename) + '.png'
1612 )
1613 _log.debug('[%s] -> [%s]', filename, qr_filename)
1614 qr = pyqrcode.create(text, encoding = 'utf8')
1615 if verbose:
1616 print('input file:', filename)
1617 print('output file:', qr_filename)
1618 print('text to encode:', text)
1619 print(qr.terminal())
1620 qr.png(qr_filename, quiet_zone = 1)
1621 return qr_filename
1622
1623 #===========================================================================
1624 # main
1625 #---------------------------------------------------------------------------
1626 if __name__ == '__main__':
1627
1628 if len(sys.argv) < 2:
1629 sys.exit()
1630
1631 if sys.argv[1] != 'test':
1632 sys.exit()
1633
1634 # for testing:
1635 logging.basicConfig(level = logging.DEBUG)
1636 from Gnumed.pycommon import gmI18N
1637 gmI18N.activate_locale()
1638 gmI18N.install_domain()
1639
1640 #-----------------------------------------------------------------------
1642
1643 tests = [
1644 [None, False],
1645
1646 ['', False],
1647 [' 0 ', True, 0],
1648
1649 [0, True, 0],
1650 [0.0, True, 0],
1651 [.0, True, 0],
1652 ['0', True, 0],
1653 ['0.0', True, 0],
1654 ['0,0', True, 0],
1655 ['00.0', True, 0],
1656 ['.0', True, 0],
1657 [',0', True, 0],
1658
1659 [0.1, True, decimal.Decimal('0.1')],
1660 [.01, True, decimal.Decimal('0.01')],
1661 ['0.1', True, decimal.Decimal('0.1')],
1662 ['0,1', True, decimal.Decimal('0.1')],
1663 ['00.1', True, decimal.Decimal('0.1')],
1664 ['.1', True, decimal.Decimal('0.1')],
1665 [',1', True, decimal.Decimal('0.1')],
1666
1667 [1, True, 1],
1668 [1.0, True, 1],
1669 ['1', True, 1],
1670 ['1.', True, 1],
1671 ['1,', True, 1],
1672 ['1.0', True, 1],
1673 ['1,0', True, 1],
1674 ['01.0', True, 1],
1675 ['01,0', True, 1],
1676 [' 01, ', True, 1],
1677
1678 [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')]
1679 ]
1680 for test in tests:
1681 conversion_worked, result = input2decimal(initial = test[0])
1682
1683 expected2work = test[1]
1684
1685 if conversion_worked:
1686 if expected2work:
1687 if result == test[2]:
1688 continue
1689 else:
1690 print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result))
1691 else:
1692 print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result))
1693 else:
1694 if not expected2work:
1695 continue
1696 else:
1697 print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
1698 #-----------------------------------------------------------------------
1703 #-----------------------------------------------------------------------
1705
1706 val = None
1707 print(val, coalesce(val, 'is None', 'is not None'))
1708 val = 1
1709 print(val, coalesce(val, 'is None', 'is not None'))
1710 return
1711
1712 import datetime as dt
1713 print(coalesce(initial = dt.datetime.now(), template_initial = '-- %s --', function_initial = ('strftime', '%Y-%m-%d')))
1714
1715 print('testing coalesce()')
1716 print("------------------")
1717 tests = [
1718 [None, 'something other than <None>', None, None, 'something other than <None>'],
1719 ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'],
1720 ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'],
1721 ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'],
1722 [None, 'initial value was None', 'template_initial: %s', None, 'initial value was None'],
1723 [None, 'initial value was None', 'template_initial: %%(abc)s', None, 'initial value was None']
1724 ]
1725 passed = True
1726 for test in tests:
1727 result = coalesce (
1728 initial = test[0],
1729 instead = test[1],
1730 template_initial = test[2],
1731 template_instead = test[3]
1732 )
1733 if result != test[4]:
1734 print("ERROR")
1735 print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3]))
1736 print("expected:", test[4])
1737 print("received:", result)
1738 passed = False
1739
1740 if passed:
1741 print("passed")
1742 else:
1743 print("failed")
1744 return passed
1745 #-----------------------------------------------------------------------
1747 print('testing capitalize() ...')
1748 success = True
1749 pairs = [
1750 # [original, expected result, CAPS mode]
1751 ['Boot', 'Boot', CAPS_FIRST_ONLY],
1752 ['boot', 'Boot', CAPS_FIRST_ONLY],
1753 ['booT', 'Boot', CAPS_FIRST_ONLY],
1754 ['BoOt', 'Boot', CAPS_FIRST_ONLY],
1755 ['boots-Schau', 'Boots-Schau', CAPS_WORDS],
1756 ['boots-sChau', 'Boots-Schau', CAPS_WORDS],
1757 ['boot camp', 'Boot Camp', CAPS_WORDS],
1758 ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES],
1759 ['häkkönen', 'Häkkönen', CAPS_NAMES],
1760 ['McBurney', 'McBurney', CAPS_NAMES],
1761 ['mcBurney', 'McBurney', CAPS_NAMES],
1762 ['blumberg', 'Blumberg', CAPS_NAMES],
1763 ['roVsing', 'RoVsing', CAPS_NAMES],
1764 ['Özdemir', 'Özdemir', CAPS_NAMES],
1765 ['özdemir', 'Özdemir', CAPS_NAMES],
1766 ]
1767 for pair in pairs:
1768 result = capitalize(pair[0], pair[2])
1769 if result != pair[1]:
1770 success = False
1771 print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1]))
1772
1773 if success:
1774 print("... SUCCESS")
1775
1776 return success
1777 #-----------------------------------------------------------------------
1779 print("testing import_module_from_directory()")
1780 path = sys.argv[1]
1781 name = sys.argv[2]
1782 try:
1783 mod = import_module_from_directory(module_path = path, module_name = name)
1784 except:
1785 print("module import failed, see log")
1786 return False
1787
1788 print("module import succeeded", mod)
1789 print(dir(mod))
1790 return True
1791 #-----------------------------------------------------------------------
1795 #-----------------------------------------------------------------------
1797 print("testing gmPaths()")
1798 print("-----------------")
1799 paths = gmPaths(wx=None, app_name='gnumed')
1800 print("user config dir:", paths.user_config_dir)
1801 print("system config dir:", paths.system_config_dir)
1802 print("local base dir:", paths.local_base_dir)
1803 print("system app data dir:", paths.system_app_data_dir)
1804 print("working directory :", paths.working_dir)
1805 print("temp directory :", paths.tmp_dir)
1806 #-----------------------------------------------------------------------
1808 print("testing none_if()")
1809 print("-----------------")
1810 tests = [
1811 [None, None, None],
1812 ['a', 'a', None],
1813 ['a', 'b', 'a'],
1814 ['a', None, 'a'],
1815 [None, 'a', None],
1816 [1, 1, None],
1817 [1, 2, 1],
1818 [1, None, 1],
1819 [None, 1, None]
1820 ]
1821
1822 for test in tests:
1823 if none_if(value = test[0], none_equivalent = test[1]) != test[2]:
1824 print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2]))
1825
1826 return True
1827 #-----------------------------------------------------------------------
1829 tests = [
1830 [True, 'Yes', 'Yes', 'Yes'],
1831 [False, 'OK', 'not OK', 'not OK']
1832 ]
1833 for test in tests:
1834 if bool2str(test[0], test[1], test[2]) != test[3]:
1835 print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3]))
1836
1837 return True
1838 #-----------------------------------------------------------------------
1840
1841 print(bool2subst(True, 'True', 'False', 'is None'))
1842 print(bool2subst(False, 'True', 'False', 'is None'))
1843 print(bool2subst(None, 'True', 'False', 'is None'))
1844 #-----------------------------------------------------------------------
1846 print(get_unique_filename())
1847 print(get_unique_filename(prefix='test-'))
1848 print(get_unique_filename(suffix='tst'))
1849 print(get_unique_filename(prefix='test-', suffix='tst'))
1850 print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
1851 #-----------------------------------------------------------------------
1853 print("testing size2str()")
1854 print("------------------")
1855 tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000]
1856 for test in tests:
1857 print(size2str(test))
1858 #-----------------------------------------------------------------------
1860
1861 test = """
1862 second line\n
1863 3rd starts with tab \n
1864 4th with a space \n
1865
1866 6th
1867
1868 """
1869 print(unwrap(text = test, max_length = 25))
1870 #-----------------------------------------------------------------------
1872 test = 'line 1\nline 2\nline 3'
1873
1874 print("wrap 5-6-7 initial 0, subsequent 0")
1875 print(wrap(test, 5))
1876 print()
1877 print(wrap(test, 6))
1878 print()
1879 print(wrap(test, 7))
1880 print("-------")
1881 input()
1882 print("wrap 5 initial 1-1-3, subsequent 1-3-1")
1883 print(wrap(test, 5, ' ', ' '))
1884 print()
1885 print(wrap(test, 5, ' ', ' '))
1886 print()
1887 print(wrap(test, 5, ' ', ' '))
1888 print("-------")
1889 input()
1890 print("wrap 6 initial 1-1-3, subsequent 1-3-1")
1891 print(wrap(test, 6, ' ', ' '))
1892 print()
1893 print(wrap(test, 6, ' ', ' '))
1894 print()
1895 print(wrap(test, 6, ' ', ' '))
1896 print("-------")
1897 input()
1898 print("wrap 7 initial 1-1-3, subsequent 1-3-1")
1899 print(wrap(test, 7, ' ', ' '))
1900 print()
1901 print(wrap(test, 7, ' ', ' '))
1902 print()
1903 print(wrap(test, 7, ' ', ' '))
1904 #-----------------------------------------------------------------------
1906 print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2])))
1907 print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
1908 #-----------------------------------------------------------------------
1910 print(u_link_symbol * 10)
1911 #-----------------------------------------------------------------------
1913 print(xml_escape_string('<'))
1914 print(xml_escape_string('>'))
1915 print(xml_escape_string('&'))
1916 #-----------------------------------------------------------------------
1918 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
1919 tests.append(' '.join(tests))
1920 for test in tests:
1921 print('%s:' % test, tex_escape_string(test))
1922
1923 #-----------------------------------------------------------------------
1925 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
1926 tests.append(' '.join(tests))
1927 tests.append('C:\Windows\Programme\System 32\lala.txt')
1928 tests.extend([
1929 'should be identical',
1930 'text *some text* text',
1931 """A List
1932 ======
1933
1934 1. 1
1935 2. 2
1936
1937 3. ist-list
1938 1. more
1939 2. noch was ü
1940 #. nummer x"""
1941 ])
1942 for test in tests:
1943 print('==================================================')
1944 print('raw:')
1945 print(test)
1946 print('---------')
1947 print('ReST 2 LaTeX:')
1948 latex = rst2latex_snippet(test)
1949 print(latex)
1950 if latex.strip() == test.strip():
1951 print('=> identical')
1952 print('---------')
1953 print('tex_escape_string:')
1954 print(tex_escape_string(test))
1955 input()
1956
1957 #-----------------------------------------------------------------------
1959 tests = [
1960 'one line, no embedded line breaks ',
1961 'one line\nwith embedded\nline\nbreaks\n '
1962 ]
1963 for test in tests:
1964 print('as list:')
1965 print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True))
1966 print('as string:')
1967 print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False))
1968 tests = [
1969 ['list', 'without', 'empty', 'trailing', 'lines'],
1970 ['list', 'with', 'empty', 'trailing', 'lines', '', ' ', '']
1971 ]
1972 for test in tests:
1973 print('as list:')
1974 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True))
1975 print('as string:')
1976 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
1977 #-----------------------------------------------------------------------
1979 tests = [
1980 r'abc.exe',
1981 r'\abc.exe',
1982 r'c:\abc.exe',
1983 r'c:\d\abc.exe',
1984 r'/home/ncq/tmp.txt',
1985 r'~/tmp.txt',
1986 r'./tmp.txt',
1987 r'./.././tmp.txt',
1988 r'tmp.txt'
1989 ]
1990 for t in tests:
1991 print("[%s] -> [%s]" % (t, fname_stem(t)))
1992 #-----------------------------------------------------------------------
1994 print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
1995
1996 #-----------------------------------------------------------------------
1998 d1 = {}
1999 d2 = {}
2000 d1[1] = 1
2001 d1[2] = 2
2002 d1[3] = 3
2003 # 4
2004 d1[5] = 5
2005
2006 d2[1] = 1
2007 d2[2] = None
2008 # 3
2009 d2[4] = 4
2010
2011 #compare_dict_likes(d1, d2)
2012
2013 d1 = {1: 1, 2: 2}
2014 d2 = {1: 1, 2: 2}
2015
2016 #compare_dict_likes(d1, d2, 'same1', 'same2')
2017 print(format_dict_like(d1, tabular = False))
2018 print(format_dict_like(d1, tabular = True))
2019 #print(format_dict_like(d2))
2020
2021 #-----------------------------------------------------------------------
2023 d1 = {}
2024 d2 = {}
2025 d1[1] = 1
2026 d1[2] = 2
2027 d1[3] = 3
2028 # 4
2029 d1[5] = 5
2030
2031 d2[1] = 1
2032 d2[2] = None
2033 # 3
2034 d2[4] = 4
2035
2036 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2037
2038 d1 = {1: 1, 2: 2}
2039 d2 = {1: 1, 2: 2}
2040
2041 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2042
2043 #-----------------------------------------------------------------------
2045 rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
2046
2047 #-----------------------------------------------------------------------
2049 #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx'))
2050 print(rm_dir_content('/tmp/user/1000/tmp'))
2051
2052 #-----------------------------------------------------------------------
2054 tests = [
2055 ('', '', ''),
2056 ('a', 'a', ''),
2057 ('\.br\MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\')
2058 ]
2059 for test in tests:
2060 text, prefix, expect = test
2061 result = strip_prefix(text, prefix)
2062 if result == expect:
2063 continue
2064 print('test failed:', test)
2065 print('result:', result)
2066 #-----------------------------------------------------------------------
2068 tst = [
2069 ('123', 1),
2070 ('123', 2),
2071 ('123', 3),
2072 ('123', 4),
2073 ('', 1),
2074 ('1', 1),
2075 ('12', 1),
2076 ('', 2),
2077 ('1', 2),
2078 ('12', 2),
2079 ('123', 2)
2080 ]
2081 for txt, lng in tst:
2082 print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
2083 #-----------------------------------------------------------------------
2085 tests = [
2086 '/tmp/test.txt',
2087 '/tmp/ test.txt',
2088 '/tmp/ tes\\t.txt',
2089 'test'
2090 ]
2091 for test in tests:
2092 print (test, fname_sanitize(test))
2093
2094 #-----------------------------------------------------------------------
2097
2098 #-----------------------------------------------------------------------
2099 #test_coalesce()
2100 #test_capitalize()
2101 #test_import_module()
2102 #test_mkdir()
2103 #test_gmPaths()
2104 #test_none_if()
2105 #test_bool2str()
2106 #test_bool2subst()
2107 #test_get_unique_filename()
2108 #test_size2str()
2109 #test_wrap()
2110 #test_input2decimal()
2111 #test_input2int()
2112 #test_unwrap()
2113 #test_md5()
2114 #test_unicode()
2115 #test_xml_escape()
2116 #test_strip_trailing_empty_lines()
2117 #test_fname_stem()
2118 #test_tex_escape()
2119 #test_rst2latex_snippet()
2120 #test_dir_is_empty()
2121 #test_compare_dicts()
2122 #test_rm_dir()
2123 #test_rm_dir_content()
2124 #test_strip_prefix()
2125 #test_shorten_text()
2126 #test_format_compare_dicts()
2127 #test_fname_sanitize()
2128 #test_create_qrcode()
2129
2130 #===========================================================================
2131
| Home | Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Fri Jan 25 02:55:27 2019 | http://epydoc.sourceforge.net |