From 429d9ba2ff9c2cf4f34dd87eac89ac31f7252a91 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 30 Nov 2022 17:34:00 +0100 Subject: [PATCH 1/2] dvc: Add utils for saving experiments. --- src/dvclive/dvc.py | 64 +++++++++++++++++++++++++++++++++++++++++++++- tests/test_dvc.py | 63 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 tests/test_dvc.py diff --git a/src/dvclive/dvc.py b/src/dvclive/dvc.py index eb5f005d..806c970a 100644 --- a/src/dvclive/dvc.py +++ b/src/dvclive/dvc.py @@ -1,6 +1,13 @@ +import logging import os +from random import choice -from . import env +from dvclive import env +from dvclive.serialize import dump_yaml + +logging.basicConfig() +logger = logging.getLogger("dvclive") +logger.setLevel(os.getenv(env.DVCLIVE_LOGLEVEL, "INFO").upper()) _CHECKPOINT_SLEEP = 0.1 @@ -46,3 +53,58 @@ def make_checkpoint(): os.fsync(fobj.fileno()) while os.path.exists(signal_file): sleep(_CHECKPOINT_SLEEP) + + +def get_dvc_repo(): + # noqa pylint: disable=unused-import + try: + import dvc # noqa: F401 + except ImportError: + return None + + from dvc.exceptions import NotDvcRepoError + from dvc.repo import Repo + from dvc.scm import SCMError + + try: + return Repo() + except (NotDvcRepoError, SCMError): + logger.warning( + "Can't save experiment without a DVC Repo." + "\nYou can create a DVC Repo by calling `dvc init`." + ) + return None + + +def random_exp_name(dvc_repo, baseline_rev): + from dvc.repo.experiments.exceptions import ExperimentExistsError + from dvc.repo.experiments.refs import ExpRefInfo + + # fmt: off + NOUNS = ('abac', 'abbs', 'aces', 'acid', 'acne', 'acre', 'acts', 'ados', 'adze', 'afro', 'agas', 'aged', 'ages', 'agio', 'agma', 'airs', 'airt', 'aits', 'akes', 'alap', 'albs', 'alga', 'ally', 'alto', 'amah', 'ambo', 'amie', 'amyl', 'ankh', 'anus', 'apex', 'aqua', 'arcs', 'areg', 'aria', 'aril', 'arks', 'army', 'auks', 'aune', 'aura', 'awls', 'awns', 'axon', 'azan', 'baby', 'bade', 'bael', 'bags', 'bait', 'ball', 'banc', 'bang', 'bani', 'barb', 'bark', 'bate', 'bats', 'bawl', 'beak', 'bean', 'beep', 'belt', 'berk', 'beth', 'bias', 'bice', 'bids', 'bind', 'bise', 'bish', 'bite', 'boar', 'boat', 'body', 'boff', 'bold', 'boll', 'bolo', 'bomb', 'bond', 'book', 'boor', 'boot', 'bort', 'bosk', 'bots', 'bott', 'bout', 'bras', 'bree', 'brig', 'brio', 'buck', 'buhl', 'bump', 'bunk', 'bunt', 'buoy', 'byes', 'byte', 'cane', 'cant', 'caps', 'care', 'cart', 'cats', 'cedi', 'ceps', 'cere', 'chad', 'cham', 'chat', 'chay', 'chic', 'chin', 'chis', 'chiv', 'choc', 'chow', 'chum', 'ciao', 'cigs', 'clay', 'clip', 'clog', 'coal', 'coat', 'coca', 'cock', 'code', 'coed', 'cogs', 'coho', 'cole', 'cols', 'colt', 'coma', 'conk', 'cons', 'cony', 'coof', 'cook', 'cool', 'coos', 'corm', 'cors', 'coth', 'cows', 'coze', 'crag', 'craw', 'cree', 'crib', 'cuds', 'cull', 'cult', 'curb', 'curn', 'curs', 'cusp', 'cuss', 'cwms', 'cyma', 'cyst', 'dabs', 'dado', 'daff', 'dais', 'daks', 'damn', 'dams', 'darg', 'dart', 'data', 'dawk', 'dawn', 'daws', 'daze', 'dead', 'dean', 'debs', 'debt', 'deep', 'dees', 'dele', 'delf', 'dent', 'deys', 'dhow', 'digs', 'dirk', 'dita', 'diva', 'divs', 'doek', 'doge', 'dogs', 'dogy', 'dohs', 'doit', 'dole', 'doll', 'dolt', 'dona', 'dook', 'door', 'dops', 'doss', 'doxy', 'drab', 'drop', 'drum', 'duad', 'duct', 'duff', 'duke', 'dunk', 'dunt', 'ears', 'ease', 'eggs', 'eild', 'emeu', 'emus', 'envy', 'epha', 'eric', 'erns', 'esne', 'esse', 'ewes', 'expo', 'eyas', 'eyot', 'eyry', 'fare', 'farl', 'farm', 'feds', 'feel', 'fees', 'feme', 'fess', 'fibs', 'fids', 'fils', 'firm', 'fish', 'flab', 'flap', 'flea', 'flew', 'flex', 'flip', 'flit', 'flus', 'flux', 'foil', 'fond', 'food', 'fool', 'ford', 'fore', 'frit', 'friz', 'froe', 'funs', 'furl', 'fuss', 'fuzz', 'gaby', 'gaff', 'gale', 'gang', 'gaol', 'gape', 'gash', 'gaur', 'gays', 'gaze', 'gear', 'genu', 'gest', 'geum', 'ghat', 'gigs', 'gimp', 'gird', 'girl', 'glee', 'glen', 'glia', 'glop', 'gnat', 'goad', 'goaf', 'gobs', 'gonk', 'good', 'goos', 'gore', 'gram', 'gray', 'grig', 'grip', 'grot', 'grub', 'gude', 'gula', 'gulf', 'guns', 'gust', 'gyms', 'gyps', 'gyro', 'hack', 'haet', 'hajj', 'hake', 'half', 'halm', 'hard', 'harl', 'hask', 'hate', "he'd", 'heck', 'heel', 'heir', 'help', 'hems', 'here', 'hill', 'hips', 'hits', 'hobo', 'hock', 'hogs', 'hold', 'holy', 'hood', 'hoot', 'hope', 'horn', 'hose', 'hour', 'hows', 'huck', 'hugs', 'huia', 'hulk', 'hull', 'hunk', 'hunt', 'huts', 'hymn', 'ibex', 'ices', 'iglu', 'impi', 'inks', 'inti', 'ions', 'iota', 'iron', 'jabs', 'jags', 'jake', 'jass', 'jato', 'jaws', 'jean', 'jeer', 'jerk', 'jest', 'jiao', 'jigs', 'jill', 'jinn', 'jird', 'jive', 'jock', 'joey', 'jogs', 'joss', 'jota', 'jots', 'juba', 'jube', 'judo', 'jump', 'junk', 'jura', 'juts', 'jynx', 'kago', 'kail', 'kaka', 'kale', 'kana', 'keek', 'keep', 'kefs', 'kegs', 'kerf', 'kern', 'keys', 'kibe', 'kick', 'kids', 'kifs', 'kill', 'kina', 'kind', 'kine', 'kite', 'kiwi', 'knap', 'knit', 'koas', 'kobs', 'kyat', 'lack', 'lahs', 'lair', 'lama', 'lamb', 'lame', 'lats', 'lava', 'lays', 'leaf', 'leak', 'leas', 'lees', 'leks', 'leno', 'libs', 'lich', 'lick', 'lien', 'lier', 'lieu', 'life', 'lift', 'limb', 'line', 'link', 'linn', 'lira', 'loft', 'loge', 'loir', 'long', 'loof', 'look', 'loot', 'lore', 'loss', 'lots', 'loup', 'love', 'luce', 'ludo', 'luke', 'lulu', 'lure', 'lush', 'magi', 'maid', 'main', 'mako', 'male', 'mana', 'many', 'mart', 'mash', 'mast', 'mate', 'math', 'mats', 'matt', 'maul', 'maya', 'mays', 'meal', 'mean', 'meed', 'mela', 'mene', 'mere', 'merk', 'mesh', 'mete', 'mice', 'milo', 'mime', 'mina', 'mine', 'mirk', 'miss', 'mobs', 'moit', 'mold', 'molt', 'mome', 'moms', 'mong', 'monk', 'moot', 'mope', 'more', 'morn', 'mows', 'moxa', 'much', 'mung', 'mush', 'muss', 'myth', 'name', 'nard', 'nark', 'nave', 'navy', 'neck', 'newt', 'nibs', 'nims', 'nine', 'nock', 'noil', 'noma', 'nosh', 'nowt', 'nuke', 'oafs', 'oast', 'oats', 'obit', 'odor', 'okra', 'omer', 'oner', 'ones', 'orcs', 'ords', 'orfe', 'orgy', 'orle', 'ossa', 'outs', 'over', 'owls', 'pail', 'pall', 'palp', 'pams', 'pang', 'pans', 'pant', 'paps', 'pate', 'pats', 'paws', 'pear', 'peba', 'pech', 'pecs', 'peel', 'peer', 'pees', 'pein', 'peri', 'perv', 'phon', 'pice', 'pita', 'pith', 'play', 'plop', 'plot', 'plow', 'plug', 'plum', 'polo', 'pomp', 'pond', 'pons', 'pony', 'poof', 'poon', 'pope', 'porn', 'poss', 'pots', 'pour', 'prad', 'prat', 'prep', 'prob', 'prof', 'prow', 'puck', 'puds', 'puke', 'puku', 'pump', 'puns', 'pupa', 'purl', 'pyre', 'quad', 'quay', 'quey', 'quiz', 'raid', 'rail', 'rain', 'raja', 'rale', 'rams', 'rand', 'rant', 'raps', 'rasp', 'razz', 'rede', 'reef', 'reif', 'rein', 'repp', 'rial', 'ribs', 'rick', 'rift', 'rill', 'rime', 'rims', 'ring', 'rins', 'rise', 'rite', 'rits', 'roam', 'robe', 'rods', 'roma', 'rook', 'rort', 'rotl', 'roup', 'roux', 'rube', 'rubs', 'ruby', 'rues', 'rugs', 'ruin', 'runs', 'ryas', 'sack', 'sacs', 'saga', 'sail', 'sale', 'salp', 'salt', 'sand', 'sang', 'sash', 'saut', 'says', 'scab', 'scow', 'scud', 'scup', 'scut', 'seal', 'seam', 'sech', 'seed', 'seep', 'seer', 'self', 'sena', 'send', 'sera', 'sere', 'sext', 'shad', 'shag', 'shah', 'sham', 'shay', 'shes', 'ship', 'shoe', 'sick', 'sida', 'sign', 'sike', 'sima', 'sine', 'sing', 'sinh', 'sink', 'sins', 'site', 'size', 'skat', 'skin', 'skip', 'skis', 'slaw', 'sled', 'slew', 'sley', 'slob', 'slue', 'slug', 'smut', 'snap', 'snib', 'snip', 'snob', 'snog', 'snot', 'snow', 'snub', 'snug', 'soft', 'soja', 'soke', 'song', 'sons', 'sook', 'sorb', 'sori', 'souk', 'soul', 'sous', 'soya', 'spit', 'stay', 'stew', 'stir', 'stob', 'stud', 'suck', 'suds', 'suer', 'suit', 'sumo', 'sums', 'sups', 'suqs', 'suss', 'sway', 'syce', 'synd', 'taal', 'tach', 'taco', 'tads', 'taka', 'tale', 'tamp', 'tams', 'tang', 'tans', 'tape', 'tare', 'taro', 'tarp', 'tart', 'tass', 'taus', 'teat', 'teds', 'teff', 'tegu', 'tell', 'term', 'thar', 'thaw', 'tics', 'tier', 'tiff', 'tils', 'tilt', 'tint', 'tipi', 'tire', 'tirl', 'tits', 'toby', 'tods', 'toea', 'toff', 'toga', 'toil', 'toke', 'tola', 'tole', 'tomb', 'toms', 'torc', 'tors', 'tort', 'tosh', 'tote', 'tret', 'trey', 'trio', 'trug', 'tuck', 'tugs', 'tule', 'tune', 'tuns', 'tuts', 'tyke', 'tyne', 'typo', 'ulna', 'umbo', 'unau', 'unit', 'upas', 'urea', 'user', 'uvea', 'vacs', 'vane', 'vang', 'vans', 'vara', 'vase', 'veep', 'veer', 'vega', 'veil', 'vela', 'vent', 'vies', 'view', 'vina', 'vine', 'vise', 'vlei', 'volt', 'vows', 'wads', 'waft', 'wage', 'wain', 'walk', 'want', 'wart', 'wave', 'waws', 'weal', 'wean', 'weds', 'weep', 'weft', 'weir', 'weka', 'weld', 'wens', 'weys', 'whap', 'whey', 'whin', 'whit', 'whop', 'wide', 'wife', 'wind', 'wine', 'wino', 'wins', 'wire', 'wise', 'woes', 'wont', 'wool', 'work', 'worm', 'wort', 'wuss', 'yack', 'yank', 'yapp', 'yard', 'yate', 'yawl', 'yegg', 'yell', 'yeuk', 'yews', 'yips', 'yobs', 'yogi', 'yoke', 'yolk', 'yoni', 'zack', 'zags', 'zest', 'zhos', 'zigs', 'zila', 'zips', 'ziti', 'zoea', 'zone', 'zoon') # noqa: E501 + ADJECTIVES = ('about', 'above', 'abuzz', 'acerb', 'acock', 'acold', 'acred', 'added', 'addle', 'adept', 'adult', 'adunc', 'adust', 'afoul', 'after', 'agape', 'agaze', 'agile', 'aging', 'agley', 'aglow', 'ahead', 'ahull', 'aided', 'alary', 'algal', 'alike', 'alive', 'alone', 'aloof', 'alpha', 'amber', 'amiss', 'amort', 'ample', 'amuck', 'angry', 'anile', 'apeak', 'apish', 'arced', 'areal', 'armed', 'aroid', 'ashen', 'aspen', 'astir', 'atilt', 'atrip', 'aulic', 'aural', 'awash', 'awful', 'awing', 'awned', 'axile', 'azoic', 'azure', 'baggy', 'baked', 'balky', 'bally', 'balmy', 'banal', 'bandy', 'bardy', 'bared', 'barer', 'barky', 'basal', 'based', 'baser', 'basic', 'batty', 'bawdy', 'beady', 'beaky', 'beamy', 'beaut', 'beefy', 'beery', 'beige', 'bendy', 'bifid', 'bijou', 'biped', 'birch', 'bitty', 'blame', 'bland', 'blank', 'blear', 'blest', 'blind', 'blond', 'blown', 'blowy', 'bluer', 'bluff', 'blunt', 'boned', 'bonny', 'boozy', 'bored', 'boric', 'bosky', 'bosom', 'bound', 'bovid', 'bowed', 'boxed', 'braky', 'brash', 'brief', 'briny', 'brisk', 'broad', 'broch', 'brood', 'brown', 'brute', 'buggy', 'bulgy', 'bumpy', 'burly', 'burnt', 'burry', 'bushy', 'busty', 'butch', 'buxom', 'cadgy', 'cagey', 'calmy', 'campy', 'canny', 'caped', 'cased', 'catty', 'cauld', 'cedar', 'cered', 'ceric', 'chary', 'cheap', 'cheek', 'chewy', 'chief', 'chill', 'chirk', 'choky', 'cissy', 'civil', 'cleft', 'coaly', 'cocky', 'color', 'comfy', 'comic', 'compo', 'conic', 'couth', 'coxal', 'crack', 'crank', 'crash', 'crass', 'crisp', 'cronk', 'cross', 'crude', 'cruel', 'crumb', 'cured', 'curly', 'curst', 'cushy', 'cutty', 'cynic', 'dated', 'dazed', 'dedal', 'deism', 'diazo', 'dicey', 'dingy', 'direr', 'dirty', 'dishy', 'dizzy', 'dolce', 'doped', 'dopey', 'dormy', 'dorty', 'dosed', 'dotal', 'dotty', 'dowdy', 'dowie', 'downy', 'dozen', 'drawn', 'dread', 'drear', 'dress', 'dried', 'ducky', 'duddy', 'dummy', 'dumpy', 'duple', 'dural', 'dusky', 'dusty', 'dutch', 'dying', 'eager', 'eaten', 'ebony', 'edged', 'eerie', 'eight', 'elder', 'elect', 'elfin', 'elite', 'empty', 'enate', 'enemy', 'epoxy', 'erect', 'ethic', 'every', 'extra', 'faced', 'faery', 'faint', 'famed', 'fancy', 'farci', 'fatal', 'fated', 'fatty', 'fazed', 'fecal', 'felon', 'fenny', 'ferny', 'fetal', 'fetid', 'fewer', 'fiery', 'fifty', 'filar', 'filmy', 'final', 'fined', 'finer', 'finny', 'fired', 'first', 'fishy', 'fixed', 'fizzy', 'flaky', 'flamy', 'flash', 'flawy', 'fleet', 'flory', 'flown', 'fluid', 'fluky', 'flush', 'focal', 'foggy', 'folio', 'forky', 'forte', 'forty', 'found', 'frail', 'frank', 'freed', 'freer', 'fresh', 'fried', 'front', 'frore', 'fuggy', 'funky', 'funny', 'furry', 'fusil', 'fussy', 'fuzzy', 'gabby', 'gamer', 'gamey', 'gamic', 'gammy', 'garni', 'gauge', 'gaunt', 'gauzy', 'gawky', 'gawsy', 'gemmy', 'genal', 'genic', 'ghast', 'gimpy', 'girly', 'glare', 'glary', 'glial', 'glued', 'gluey', 'godly', 'gooey', 'goofy', 'goosy', 'gouty', 'grade', 'grand', 'grapy', 'grave', 'gross', 'group', 'gruff', 'guest', 'gules', 'gulfy', 'gummy', 'gushy', 'gusty', 'gutsy', 'gutta', 'gypsy', 'gyral', 'hadal', 'hammy', 'handy', 'hardy', 'hasty', 'hated', 'hazel', 'heady', 'heapy', 'hefty', 'heigh', 'hempy', 'herby', 'hexed', 'hi-fi', 'hilly', 'hired', 'hoary', 'holey', 'honey', 'hooly', 'horny', 'hoven', 'huger', 'hulky', 'humid', 'hunky', 'hyoid', 'idled', 'iliac', 'inane', 'incog', 'inert', 'inner', 'inter', 'iodic', 'ionic', 'irate', 'irony', 'itchy', 'jaggy', 'jammy', 'japan', 'jazzy', 'jerky', 'jetty', 'joint', 'jowly', 'juicy', 'jumpy', 'jural', 'kacha', 'kaput', 'kempt', 'keyed', 'kinky', 'known', 'kooky', 'kraal', 'laced', 'laigh', 'lairy', 'lamer', 'lardy', 'larky', 'lated', 'later', 'lathy', 'leady', 'leafy', 'leaky', 'leary', 'least', 'ledgy', 'leery', 'legal', 'leggy', 'lento', 'level', 'licht', 'licit', 'liege', 'light', 'liked', 'liney', 'lippy', 'lived', 'livid', 'loamy', 'loath', 'lobar', 'local', 'loony', 'loose', 'loral', 'losel', 'lousy', 'loved', 'lower', 'lowly', 'lowse', 'loyal', 'lucid', 'lucky', 'lumpy', 'lunar', 'lurid', 'lushy', 'lying', 'lyric', 'macho', 'macro', 'magic', 'major', 'malar', 'mangy', 'manky', 'manly', 'mardy', 'massy', 'mated', 'matte', 'mauve', 'mazed', 'mealy', 'meaty', 'medal', 'melic', 'mesic', 'mesne', 'messy', 'metal', 'miffy', 'milky', 'mined', 'minim', 'minor', 'minus', 'mired', 'mirky', 'misty', 'mixed', 'modal', 'model', 'moire', 'molar', 'moldy', 'moody', 'moony', 'mopey', 'moral', 'mossy', 'mothy', 'motor', 'mousy', 'moved', 'mucid', 'mucky', 'muddy', 'muggy', 'muley', 'mural', 'murky', 'mushy', 'muted', 'muzzy', 'myoid', 'naggy', 'naive', 'naked', 'named', 'nasty', 'natal', 'naval', 'nervy', 'newsy', 'nicer', 'niffy', 'nifty', 'ninth', 'nitty', 'nival', 'noble', 'nodal', 'noisy', 'non-U', 'north', 'nosed', 'noted', 'nowed', 'nubby', 'oaken', 'oared', 'oaten', 'obese', 'ocher', 'ochre', 'often', 'ohmic', 'oiled', 'olden', 'older', 'oleic', 'olive', 'optic', 'ortho', 'osmic', 'other', 'outer', 'ovoid', 'owing', 'owned', 'paced', 'pagan', 'paled', 'paler', 'pally', 'paper', 'pappy', 'parky', 'party', 'pasty', 'pavid', 'pawky', 'peaky', 'pearl', 'peart', 'peaty', 'pedal', 'peppy', 'perdu', 'perky', 'pesky', 'phony', 'piano', 'picky', 'piled', 'piney', 'pious', 'pique', 'pithy', 'platy', 'plump', 'plush', 'podgy', 'potty', 'power', 'prest', 'pricy', 'prima', 'prime', 'print', 'privy', 'prize', 'prone', 'proof', 'prosy', 'proud', 'proxy', 'pseud', 'pucka', 'pudgy', 'puffy', 'pukka', 'pupal', 'purer', 'pursy', 'pushy', 'pussy', 'pyoid', 'quack', 'quare', 'quasi', 'quiet', 'quits', 'rabic', 'rabid', 'radio', 'raked', 'randy', 'raped', 'rapid', 'rarer', 'raspy', 'rathe', 'ratty', 'ready', 'reedy', 'reeky', 'refer', 'regal', 'riant', 'ridgy', 'right', 'riled', 'rimed', 'rindy', 'risen', 'risky', 'ritzy', 'rival', 'riven', 'robed', 'rocky', 'roily', 'roman', 'rooky', 'ropey', 'round', 'rowdy', 'ruddy', 'ruled', 'rummy', 'runic', 'runny', 'runty', 'rural', 'rusty', 'rutty', 'sable', 'salic', 'sandy', 'sappy', 'sarky', 'sassy', 'sated', 'saved', 'savvy', 'scald', 'scaly', 'scary', 'score', 'scrap', 'sedgy', 'seely', 'seral', 'sewed', 'sexed', 'shaky', 'sharp', 'sheen', 'shier', 'shill', 'shoal', 'shock', 'shoed', 'shore', 'short', 'shyer', 'silky', 'silly', 'silty', 'sixth', 'sixty', 'skint', 'slack', 'slant', 'sleek', 'slier', 'slimy', 'slung', 'small', 'smart', 'smoky', 'snaky', 'sneak', 'snide', 'snowy', 'snuff', 'so-so', 'soapy', 'sober', 'socko', 'solar', 'soled', 'solid', 'sonic', 'sooth', 'sooty', 'soppy', 'sorer', 'sound', 'soupy', 'spent', 'spicy', 'spiky', 'spiny', 'spiry', 'splay', 'split', 'sport', 'spumy', 'squat', 'staid', 'stiff', 'still', 'stoic', 'stone', 'stony', 'store', 'stout', 'straw', 'stray', 'strip', 'stung', 'suave', 'sudsy', 'sulfa', 'sulky', 'sunny', 'super', 'sural', 'surer', 'surfy', 'surgy', 'surly', 'swell', 'swept', 'swish', 'sworn', 'tabby', 'taboo', 'tacit', 'tacky', 'tamed', 'tamer', 'tangy', 'taped', 'tardy', 'tarot', 'tarry', 'tasty', 'tatty', 'taunt', 'tawie', 'teary', 'techy', 'telic', 'tenor', 'tense', 'tenth', 'tenty', 'tepid', 'terse', 'testy', 'third', 'tidal', 'tight', 'tiled', 'timid', 'tinct', 'tined', 'tippy', 'tipsy', 'tonal', 'toned', 'tonic', 'toric', 'total', 'tough', 'toxic', 'trade', 'treed', 'treen', 'trial', 'truer', 'tubal', 'tubby', 'tumid', 'tuned', 'tutti', 'twill', 'typal', 'typed', 'typic', 'umber', 'unapt', 'unbid', 'uncut', 'undue', 'undug', 'unfed', 'unfit', 'union', 'unlet', 'unmet', 'unwed', 'unwet', 'upper', 'upset', 'urban', 'utile', 'uveal', 'vagal', 'valid', 'vapid', 'varus', 'vatic', 'veiny', 'vital', 'vivid', 'vocal', 'vogie', 'volar', 'vying', 'wacky', 'wally', 'waney', 'warty', 'washy', 'waspy', 'waste', 'waugh', 'waxen', 'webby', 'wedgy', 'weeny', 'weepy', 'weest', 'weird', 'welsh', 'wersh', 'whist', 'white', 'whity', 'whole', 'wider', 'wight', 'winey', 'wired', 'wised', 'wiser', 'withy', 'wonky', 'woods', 'woozy', 'world', 'wormy', 'worse', 'worst', 'woven', 'wrath', 'wrier', 'wrong', 'wroth', 'xeric', 'yarer', 'yolky', 'young', 'yucky', 'yummy', 'zesty', 'zingy', 'zinky', 'zippy', 'zonal') # noqa: E501 + # fmt: on + while True: + adjective = choice(ADJECTIVES) # nosec B311 + noun = choice(NOUNS) # nosec B311 + name = f"{adjective}-{noun}" + + exp_ref = ExpRefInfo(baseline_rev, name) + try: + dvc_repo.experiments._validate_new_ref(exp_ref) + except ExperimentExistsError: + continue + + return name + + +def make_dvcyaml(live): + if not os.path.exists(live.dvc_file): + dvcyaml = { + "metrics": [os.path.relpath(live.metrics_file, live.dir)], + "params": [os.path.relpath(live.params_file, live.dir)], + "plots": [os.path.relpath(live.plots_dir, live.dir)], + # TODO: Should not be needed in the near future. + "stages": {"empty": {"cmd": "empty"}}, + } + dump_yaml(dvcyaml, live.dvc_file) diff --git a/tests/test_dvc.py b/tests/test_dvc.py new file mode 100644 index 00000000..827f52ca --- /dev/null +++ b/tests/test_dvc.py @@ -0,0 +1,63 @@ +from dvc.repo import Repo +from dvc.repo.experiments.exceptions import ExperimentExistsError +from scmrepo.git import Git + +from dvclive import Live +from dvclive.dvc import get_dvc_repo, make_dvcyaml, random_exp_name +from dvclive.serialize import load_yaml + + +def test_get_dvc_repo(tmp_dir): + assert get_dvc_repo() is None + Git.init(tmp_dir) + Repo.init(tmp_dir) + assert isinstance(get_dvc_repo(), Repo) + + +def test_make_dvcyaml(tmp_dir): + live = Live() + make_dvcyaml(live) + + assert load_yaml(live.dvc_file) == { + "metrics": ["metrics.json"], + "params": ["params.yaml"], + "plots": ["plots"], + "stages": {"empty": {"cmd": "empty"}}, + } + + +def test_random_exp_name(mocker): + dvc_repo = mocker.MagicMock() + + class Validate: + exists = set() + n_calls = 0 + + def __call__(self, exp_ref): + self.n_calls += 1 + exp_ref = str(exp_ref) + if exp_ref not in self.exists: + self.exists.add(exp_ref) + else: + raise ExperimentExistsError(exp_ref) + + validate = Validate() + dvc_repo.experiments._validate_new_ref.side_effect = validate + + with mocker.patch( + "dvclive.dvc.choice", side_effect=[0, 0, 0, 0, 1, 1, 0, 0] + ): + name = random_exp_name(dvc_repo, "foo") + assert name == "0-0" + assert validate.n_calls == 1 + + # First try fails with exists error + # So 2 calls are needed + name = random_exp_name(dvc_repo, "foo") + assert name == "1-1" + assert validate.n_calls == 3 + + # Doesn't fail because has a different baseline_rev + name = random_exp_name(dvc_repo, "bar") + assert name == "0-0" + assert validate.n_calls == 4 From 82a44409b3df9b396686d9593af61d638f8e252e Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Thu, 1 Dec 2022 18:16:12 +0100 Subject: [PATCH 2/2] live: Create DVC experiment on `end`. Enable with `save_dvc_exp=True`. Defaults to `False`. Refactor `__init__` method. Split into private `_init_{component}` methods. Add `_` prefix to private properties. Use env vars from https://github.com/iterative/dvc/pull/8630 to skip Studio `start` and `done` events. Use same env vars to skip creating DVC exp. --- src/dvclive/dvc.py | 3 +- src/dvclive/env.py | 2 + src/dvclive/lightning.py | 7 +- src/dvclive/live.py | 178 +++++++++++++++++++++++++-------------- src/dvclive/report.py | 6 +- src/dvclive/studio.py | 7 +- tests/test_dvc.py | 32 ++++++- tests/test_main.py | 6 +- tests/test_report.py | 8 +- tests/test_studio.py | 35 +++++--- 10 files changed, 194 insertions(+), 90 deletions(-) diff --git a/src/dvclive/dvc.py b/src/dvclive/dvc.py index 806c970a..e70bcc10 100644 --- a/src/dvclive/dvc.py +++ b/src/dvclive/dvc.py @@ -1,3 +1,4 @@ +# pylint: disable=protected-access import logging import os from random import choice @@ -104,7 +105,5 @@ def make_dvcyaml(live): "metrics": [os.path.relpath(live.metrics_file, live.dir)], "params": [os.path.relpath(live.params_file, live.dir)], "plots": [os.path.relpath(live.plots_dir, live.dir)], - # TODO: Should not be needed in the near future. - "stages": {"empty": {"cmd": "empty"}}, } dump_yaml(dvcyaml, live.dvc_file) diff --git a/src/dvclive/env.py b/src/dvclive/env.py index 2a09ebdc..a6f8b457 100644 --- a/src/dvclive/env.py +++ b/src/dvclive/env.py @@ -5,3 +5,5 @@ STUDIO_ENDPOINT = "STUDIO_ENDPOINT" STUDIO_REPO_URL = "STUDIO_REPO_URL" STUDIO_TOKEN = "STUDIO_TOKEN" # nosec B105 +DVC_EXP_BASELINE_REV = "DVC_EXP_BASELINE_REV" +DVC_EXP_NAME = "DVC_EXP_NAME" diff --git a/src/dvclive/lightning.py b/src/dvclive/lightning.py index 83eb73b7..e51cb7b4 100644 --- a/src/dvclive/lightning.py +++ b/src/dvclive/lightning.py @@ -17,11 +17,16 @@ def __init__( dir: Optional[str] = None, # noqa pylint: disable=redefined-builtin resume: bool = False, report: Optional[str] = "auto", + save_dvc_exp: bool = False, ): super().__init__() self._prefix = prefix - self._live_init: Dict[str, Any] = {"resume": resume, "report": report} + self._live_init: Dict[str, Any] = { + "resume": resume, + "report": report, + "save_dvc_exp": save_dvc_exp, + } if dir is not None: self._live_init["dir"] = dir self._experiment = experiment diff --git a/src/dvclive/live.py b/src/dvclive/live.py index c2f2a2f0..cd163335 100644 --- a/src/dvclive/live.py +++ b/src/dvclive/live.py @@ -3,12 +3,12 @@ import os import shutil from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Set, Union from ruamel.yaml.representer import RepresenterError from . import env -from .dvc import make_checkpoint +from .dvc import get_dvc_repo, make_checkpoint, make_dvcyaml, random_exp_name from .error import ( InvalidDataTypeError, InvalidParameterTypeError, @@ -40,96 +40,138 @@ def __init__( dir: str = "dvclive", # noqa pylint: disable=redefined-builtin resume: bool = False, report: Optional[str] = "auto", + save_dvc_exp: bool = False, ): + self.summary: Dict[str, Any] = {} + self._dir: str = dir self._resume: bool = resume or env2bool(env.DVCLIVE_RESUME) - self._ended: bool = False - self.studio_url = os.getenv(env.STUDIO_REPO_URL, None) - self.studio_token = os.getenv(env.STUDIO_TOKEN, None) - self.rev = None - - if report == "auto": - if self.studio_url and self.studio_token: - report = "studio" - elif env2bool("CI") and matplotlib_installed(): - report = "md" - else: - report = "html" - else: - if report not in {None, "html", "md"}: - raise ValueError( - "`report` can only be `None`, `auto`, `html` or `md`" - ) - - self.report_mode: Optional[str] = report - self.report_file = "" - - self.summary: Dict[str, Any] = {} + self._save_dvc_exp: bool = save_dvc_exp self._step: Optional[int] = None self._metrics: Dict[str, Any] = {} self._images: Dict[str, Any] = {} - self._plots: Dict[str, Any] = {} self._params: Dict[str, Any] = {} + self._plots: Dict[str, Any] = {} - self._init_paths() + os.makedirs(self.dir, exist_ok=True) - if self.report_mode in ("html", "md"): - if not self.report_file: - self.report_file = os.path.join(self.dir, f"report.{report}") - out = Path(self.report_file).resolve() - logger.info(f"Report file (if generated): {out}") + self._report_mode: Optional[str] = report + self._init_report() if self._resume: - self._read_params() - self._step = self.read_step() - if self._step != 0: - self._step += 1 - logger.info(f"Resumed from step {self._step}") + self._init_resume() else: - self._cleanup() + self._init_cleanup() + + self._baseline_rev: Optional[str] = None + self._exp_name: Optional[str] = None + self._inside_dvc_exp: bool = False + self._dvc_repo = None + self._init_dvc() + self._studio_url: Optional[str] = None + self._studio_token: Optional[str] = None self._latest_studio_step = self.step if resume else -1 - if self.report_mode == "studio": - from scmrepo.git import Git + self._studio_events_to_skip: Set[str] = set() + self._init_studio() - self.rev = Git().get_rev() + def _init_resume(self): + self._read_params() + self._step = self.read_step() + if self._step != 0: + self._step += 1 + logger.debug(f"{self._step=}") - if not post_to_studio(self, "start", logger): - logger.warning( - "`post_to_studio` `start` event failed. " - "`studio` report cancelled." - ) - self.report_mode = None - - def _cleanup(self): + def _init_cleanup(self): for plot_type in PLOT_TYPES: shutil.rmtree( Path(self.plots_dir) / plot_type.subfolder, ignore_errors=True ) for f in (self.metrics_file, self.report_file, self.params_file): - if os.path.exists(f): + if f and os.path.exists(f): os.remove(f) - def _init_paths(self): - os.makedirs(self.dir, exist_ok=True) + def _init_dvc(self): + self._dvc_repo = get_dvc_repo() + if os.getenv(env.DVC_EXP_BASELINE_REV, None): + # `dvc exp` execution + self._baseline_rev = os.getenv(env.DVC_EXP_BASELINE_REV, "") + self._exp_name = os.getenv(env.DVC_EXP_NAME, "") + self._inside_dvc_exp = True + elif self._save_dvc_exp: + # `Python Only` execution + # TODO: How to handle `dvc repro` execution? + if self._dvc_repo is not None: + self._baseline_rev = self._dvc_repo.scm.get_rev() + self._exp_name = random_exp_name( + self._dvc_repo, self._baseline_rev + ) + make_dvcyaml(self) + + def _init_studio(self): + if not self._dvc_repo: + logger.warning("`studio` report can't be used without a DVC Repo.") + return + + self._studio_url = os.getenv(env.STUDIO_REPO_URL, None) + self._studio_token = os.getenv(env.STUDIO_TOKEN, None) + + if self._studio_url and self._studio_token: + if self._inside_dvc_exp: + logger.debug( + "Skipping `post_to_studio` `start` and `done` events." + ) + self._studio_events_to_skip.add("start") + self._studio_events_to_skip.add("done") + elif not post_to_studio(self, "start", logger): + logger.warning( + "`post_to_studio` `start` event failed. " + "`studio` report cancelled." + ) + self._studio_events_to_skip.add("start") + self._studio_events_to_skip.add("data") + self._studio_events_to_skip.add("done") + logger.debug("Skipping `studio` report.") + + def _init_report(self): + if self._report_mode == "auto": + if env2bool("CI") and matplotlib_installed(): + self._report_mode = "md" + else: + self._report_mode = "html" + elif self._report_mode not in {None, "html", "md"}: + raise ValueError( + "`report` can only be `None`, `auto`, `html` or `md`" + ) + logger.debug(f"{self._report_mode=}") @property - def dir(self): + def dir(self) -> str: return self._dir @property - def params_file(self): + def params_file(self) -> str: return os.path.join(self.dir, "params.yaml") @property - def metrics_file(self): + def metrics_file(self) -> str: return os.path.join(self.dir, "metrics.json") @property - def plots_dir(self): + def dvc_file(self) -> str: + return os.path.join(self.dir, "dvc.yaml") + + @property + def plots_dir(self) -> str: return os.path.join(self.dir, "plots") + @property + def report_file(self) -> Optional[str]: + if self._report_mode in ("html", "md"): + return os.path.join(self.dir, f"report.{self._report_mode}") + return None + @property def step(self) -> int: return self._step or 0 @@ -227,7 +269,11 @@ def make_summary(self): dump_json(self.summary, self.metrics_file, cls=NumpyEncoder) def make_report(self): - if self.report_mode == "studio": + if ( + self._studio_url + and self._studio_token + and "data" not in self._studio_events_to_skip + ): if not post_to_studio(self, "data", logger): logger.warning( "`post_to_studio` `data` event failed." @@ -235,21 +281,31 @@ def make_report(self): ) else: self._latest_studio_step = self.step - elif self.report_mode is not None: + + if self._report_mode is not None: make_report(self) - if self.report_mode == "html" and env2bool(env.DVCLIVE_OPEN): + if self._report_mode == "html" and env2bool(env.DVCLIVE_OPEN): open_file_in_browser(self.report_file) def end(self): self.make_summary() - if self.report_mode == "studio": - if not self._ended: + if self._studio_url and self._studio_token: + if "done" not in self._studio_events_to_skip: if not post_to_studio(self, "done", logger): logger.warning("`post_to_studio` `done` event failed.") - self._ended = True + self._studio_events_to_skip.add("done") else: self.make_report() + if ( + self._dvc_repo is not None + and not self._inside_dvc_exp + and self._save_dvc_exp + ): + self._dvc_repo.experiments.save( + name=self._exp_name, include_untracked=self.dir + ) + def make_checkpoint(self): if env2bool(env.DVC_CHECKPOINT): make_checkpoint() diff --git a/src/dvclive/report.py b/src/dvclive/report.py index d2900e6e..b6b92950 100644 --- a/src/dvclive/report.py +++ b/src/dvclive/report.py @@ -125,9 +125,9 @@ def make_report(live: "Live"): get_plot_renderers(plots_path / SKLearnPlot.subfolder, live) ) - if live.report_mode == "html": + if live._report_mode == "html": render_html(renderers, live.report_file, refresh_seconds=5) - elif live.report_mode == "md": + elif live._report_mode == "md": render_markdown(renderers, live.report_file) else: - raise ValueError(f"Invalid `mode` {live.report_mode}.") + raise ValueError(f"Invalid `mode` {live._report_mode}.") diff --git a/src/dvclive/studio.py b/src/dvclive/studio.py index 360f6390..0c19243e 100644 --- a/src/dvclive/studio.py +++ b/src/dvclive/studio.py @@ -1,3 +1,4 @@ +# pylint: disable=protected-access from os import getenv from dvclive.env import STUDIO_ENDPOINT @@ -46,8 +47,8 @@ def post_to_studio(live, event_type, logger) -> bool: data = { "type": event_type, - "repo_url": live.studio_url, - "rev": live.rev, + "repo_url": live._studio_url, + "rev": live._baseline_rev, "client": "dvclive", } @@ -65,7 +66,7 @@ def post_to_studio(live, event_type, logger) -> bool: json=data, headers={ "Content-type": "application/json", - "Authorization": f"token {live.studio_token}", + "Authorization": f"token {live._studio_token}", }, timeout=5, ) diff --git a/tests/test_dvc.py b/tests/test_dvc.py index 827f52ca..2adb50e6 100644 --- a/tests/test_dvc.py +++ b/tests/test_dvc.py @@ -1,9 +1,12 @@ +# pylint: disable=unused-argument,protected-access +import pytest from dvc.repo import Repo from dvc.repo.experiments.exceptions import ExperimentExistsError from scmrepo.git import Git from dvclive import Live from dvclive.dvc import get_dvc_repo, make_dvcyaml, random_exp_name +from dvclive.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME from dvclive.serialize import load_yaml @@ -22,7 +25,6 @@ def test_make_dvcyaml(tmp_dir): "metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots"], - "stages": {"empty": {"cmd": "empty"}}, } @@ -61,3 +63,31 @@ def __call__(self, exp_ref): name = random_exp_name(dvc_repo, "bar") assert name == "0-0" assert validate.n_calls == 4 + + +@pytest.mark.parametrize("save", [True, False]) +def test_exp_save_on_end(tmp_dir, mocker, save): + dvc_repo = mocker.MagicMock() + with mocker.patch("dvclive.live.get_dvc_repo", return_value=dvc_repo): + live = Live(save_dvc_exp=save) + live.end() + if save: + dvc_repo.experiments.save.assert_called_with( + name=live._exp_name, include_untracked=live.dir + ) + else: + dvc_repo.assert_not_called() + + +def test_exp_save_skip_on_env_vars(tmp_dir, monkeypatch, mocker): + monkeypatch.setenv(DVC_EXP_BASELINE_REV, "foo") + monkeypatch.setenv(DVC_EXP_NAME, "bar") + + with mocker.patch("dvclive.live.get_dvc_repo", return_value=None): + live = Live(save_dvc_exp=True) + live.end() + + assert live._dvc_repo is None + assert live._baseline_rev == "foo" + assert live._exp_name == "bar" + assert live._inside_dvc_exp diff --git a/tests/test_main.py b/tests/test_main.py index 1ed879f4..e096ff96 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -166,7 +166,7 @@ def test_cleanup(tmp_dir, html): dvclive.log_metric("m1", 1) dvclive.next_step() - html_path = tmp_dir / dvclive.report_file + html_path = tmp_dir / dvclive.dir / "report.html" if html: html_path.touch() @@ -336,15 +336,13 @@ def test_logger(tmp_dir, mocker, monkeypatch): monkeypatch.setenv(env.DVCLIVE_LOGLEVEL, "DEBUG") live = Live() - msg = "Report file (if generated)" - assert msg in logger.info.call_args[0][0] live.log_metric("foo", 0) logger.debug.assert_called_with("Logged foo: 0") live.next_step() logger.debug.assert_called_with("Step: 1") live = Live(resume=True) - logger.info.assert_called_with("Resumed from step 0") + logger.debug.assert_called_with("self._step=0") def test_make_summary_without_calling_log(tmp_dir): diff --git a/tests/test_report.py b/tests/test_report.py index eccd7942..3420c56c 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -101,19 +101,19 @@ def test_get_renderers(tmp_dir, mocker): def test_report_init(monkeypatch, mocker): monkeypatch.setenv("CI", "false") live = Live() - assert live.report_mode == "html" + assert live._report_mode == "html" monkeypatch.setenv("CI", "true") live = Live() - assert live.report_mode == "md" + assert live._report_mode == "md" mocker.patch("dvclive.live.matplotlib_installed", return_value=False) live = Live() - assert live.report_mode == "html" + assert live._report_mode == "html" for report in {None, "html", "md"}: live = Live(report=report) - assert live.report_mode == report + assert live._report_mode == report with pytest.raises(ValueError): Live(report="foo") diff --git a/tests/test_studio.py b/tests/test_studio.py index 74e6d0dc..90ff5884 100644 --- a/tests/test_studio.py +++ b/tests/test_studio.py @@ -8,9 +8,8 @@ from dvclive.plots import Metric -@pytest.mark.studio def test_post_to_studio(tmp_dir, mocker, monkeypatch): - mocker.patch("scmrepo.git.Git") + mocker.patch("dvclive.live.get_dvc_repo") mocked_response = mocker.MagicMock() mocked_response.status_code = 200 mocked_post = mocker.patch("requests.post", return_value=mocked_response) @@ -96,10 +95,8 @@ def test_post_to_studio(tmp_dir, mocker, monkeypatch): ) -@pytest.mark.studio def test_post_to_studio_failed_data_request(tmp_dir, mocker, monkeypatch): - mocker.patch("scmrepo.git.Git") - + mocker.patch("dvclive.live.get_dvc_repo") valid_response = mocker.MagicMock() valid_response.status_code = 200 mocker.patch("requests.post", return_value=valid_response) @@ -146,10 +143,8 @@ def test_post_to_studio_failed_data_request(tmp_dir, mocker, monkeypatch): ) -@pytest.mark.studio def test_post_to_studio_failed_start_request(tmp_dir, mocker, monkeypatch): - mocker.patch("scmrepo.git.Git") - + mocker.patch("dvclive.live.get_dvc_repo") mocked_response = mocker.MagicMock() mocked_response.status_code = 400 mocked_post = mocker.patch("requests.post", return_value=mocked_response) @@ -169,10 +164,8 @@ def test_post_to_studio_failed_start_request(tmp_dir, mocker, monkeypatch): assert mocked_post.call_count == 1 -@pytest.mark.studio def test_post_to_studio_end_only_once(tmp_dir, mocker, monkeypatch): - mocker.patch("scmrepo.git.Git") - + mocker.patch("dvclive.live.get_dvc_repo") valid_response = mocker.MagicMock() valid_response.status_code = 200 mocked_post = mocker.patch("requests.post", return_value=valid_response) @@ -187,3 +180,23 @@ def test_post_to_studio_end_only_once(tmp_dir, mocker, monkeypatch): assert mocked_post.call_count == 3 live.end() assert mocked_post.call_count == 3 + + +@pytest.mark.studio +def test_post_to_studio_skip_on_env_var(tmp_dir, mocker, monkeypatch): + mocker.patch("dvclive.live.get_dvc_repo") + valid_response = mocker.MagicMock() + valid_response.status_code = 200 + mocked_post = mocker.patch("requests.post", return_value=valid_response) + monkeypatch.setenv(env.STUDIO_ENDPOINT, "https://0.0.0.0") + monkeypatch.setenv(env.STUDIO_REPO_URL, "STUDIO_REPO_URL") + monkeypatch.setenv(env.STUDIO_TOKEN, "STUDIO_TOKEN") + + monkeypatch.setenv(env.DVC_EXP_BASELINE_REV, "foo") + monkeypatch.setenv(env.DVC_EXP_NAME, "bar") + + with Live() as live: + live.log_metric("foo", 1) + live.next_step() + + assert mocked_post.call_count == 1