from utils import positional_deprecated import random import numpy as np import models import models.models_utils import tasks import collections import itertools import metrics import torch import time from datautils import get_loaders @positional_deprecated def simple_evaluate( # model, args, tasks_list=[] ): """Instantiate and evaluate a model on a list of tasks. :param args: Optional[str] args for the zeroShot tasks :param tasks: list[Union[str, Task]] List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise. :return Dictionary of results """ random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) assert tasks_list != [], "No tasks specified" lm = models.get_model(args.model).create_from_arg_string({"args": args}) if args.load: print('Loading checkpoint from {}...'.format(args.load)) lm.model.load_state_dict(torch.load(args.load)) if args.wbits < 16 and not args.nearest: tick = time.time() dataloader, testloader = get_loaders( args.dataset, seed=args.seed, model=args.model, seqlen=lm.seqlen ) if 'opt' in args.model: quantizers = lm.opt_sequential(dataloader) else: quantizers = lm.bloom_sequential(dataloader) print(time.time() - tick) task_dict = tasks.get_task_dict(tasks_list) results = evaluate( lm=lm, task_dict=task_dict, seed=args.seed, num_fewshot=args.num_fewshot, ) # add info about the model and few shot config results["config"] = { "model": args.model, "num_fewshot": args.num_fewshot, "batch_size": args.batch_size, "bootstrap_iters": 1000, } return results @positional_deprecated def evaluate( lm, task_dict, seed=0, num_fewshot=0, ): """Instantiate and evaluate a model on a list of tasks. :param lm: obj Language Model :param task_dict: dict[str, Task] Dictionary of tasks. Tasks will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise. :param provide_description: bool Not implemented, and this option is deprecated and will be removed in a future version in favor of a different description providing method :param num_fewshot: int Number of examples in few-shot context :return Dictionary of results """ task_dict_items = [ (name, task) for name, task in task_dict.items() if (task.has_validation_docs() or task.has_test_docs()) ] results = collections.defaultdict(dict) versions = collections.defaultdict(dict) requests = collections.defaultdict(list) requests_origin = collections.defaultdict(list) overlaps = collections.defaultdict(list) # {task_name: contaminated_docs} # If we ever run into issues where the eval tasks don't fit in memory and we can't afford a machine with bigger # memory, we can always modify this plumbing to support that, but I didn't want to include it just yet because # over-engineering is bad (or we could make it write the requests to disk and then read them back out again # - probably using an sqlite db because of all the moving parts we have # TODO: we need unit tests & sanity checks or something to ensure that the return of `validation_docs` is stable docs = {} docs_for_decontamination = collections.defaultdict(list) # get lists of each type of request for task_name, task in task_dict_items: versions[task_name] = task.VERSION # default to test doc, fall back to val doc if validation unavailable # TODO: the test-fallback-to-val system isn't final, we should revisit it at some point if task.has_test_docs(): task_doc_func = task.test_docs task_set = "test" # Required for caching in the decontamination elif task.has_validation_docs(): task_set = "val" # Required for caching in the decontamination task_doc_func = task.validation_docs else: raise RuntimeError("Task has neither test_docs nor validation_docs") # deterministically shuffle docs and chop off the first `limit` because sometimes docs are in some kind of order task_docs = list(task_doc_func()) rnd = random.Random() rnd.seed(seed) # rnd.shuffle(task_docs) description = "" for doc_id, doc in enumerate(itertools.islice(task_docs, 0, None)): docs[(task_name, doc_id)] = doc ctx = task.fewshot_context( doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description ) reqs = task.construct_requests(doc, ctx) if not isinstance(reqs, (list, tuple)): reqs = [reqs] for i, req in enumerate(reqs): requests[req.request_type].append(req) # i: index in requests for a single task instance # doc_id: unique id that we can get back to a doc using `docs` requests_origin[req.request_type].append((i, task_name, doc, doc_id)) # all responses for each (task, doc) process_res_queue = collections.defaultdict(list) # execute each type of request for reqtype, reqs in requests.items(): # TODO: right now, this code runs multiple separate LM requests for multiple Requests differing # only in index. We could implement some kind of caching, but that would be more of a band-aid # solution. we could also implement some kind of auto-grouping here; # they should end up next to each other. print("Running", reqtype, "requests") resps = getattr(lm, reqtype)([req.args for req in reqs]) resps = [ x if req.index is None else x[req.index] for x, req in zip(resps, reqs) ] for resp, (i, task_name, doc, doc_id) in zip(resps, requests_origin[reqtype]): process_res_queue[(task_name, doc_id)].append((i, resp)) vals = collections.defaultdict(list) # unpack results and sort back in order and return control to Task for (task_name, doc_id), requests in process_res_queue.items(): requests.sort(key=lambda x: x[0]) requests = [x[1] for x in requests] task = task_dict[task_name] doc = docs[(task_name, doc_id)] metrics_dict = task.process_results(doc, requests) for metric, value in metrics_dict.items(): vals[(task_name, metric)].append(value) # aggregate results for (task_name, metric), items in vals.items(): task = task_dict[task_name] real_metric = metric # key when looking up the metric with task.aggregation if metric.endswith(decontaminate_suffix): real_metric = metric.replace( decontaminate_suffix, "" ) # decontaminated still uses the same metric results[task_name][metric] = task.aggregation()[real_metric](items) # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap # so we run them less iterations. still looking for a cleaner way to do this stderr = metrics.stderr_for_metric( metric=task.aggregation()[real_metric], bootstrap_iters=1000 ) if stderr is not None: results[task_name][metric + "_stderr"] = stderr(items) return {"results": dict(results), "versions": dict(versions)} def make_table(result_dict): """Generate table of results.""" from pytablewriter import MarkdownTableWriter, LatexTableWriter md_writer = MarkdownTableWriter() latex_writer = LatexTableWriter() md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] values = [] for k, dic in result_dict["results"].items(): version = result_dict["versions"][k] for m, v in dic.items(): if m.endswith("_stderr"): continue if m + "_stderr" in dic: se = dic[m + "_stderr"] values.append([k, version, m, "%.4f" % v, "±", "%.4f" % se]) else: values.append([k, version, m, "%.4f" % v, "", ""]) k = "" version = "" md_writer.value_matrix = values latex_writer.value_matrix = values return md_writer.dumps() decontaminate_suffix = "_decontaminate"