Module pysimt.evaluator

Classes

class Evaluator (refs, beam_metrics, filters='')

Expand source code

class Evaluator:
    def __init__(self, refs, beam_metrics, filters=''):
        # metrics: list of upper-case beam-search metrics
        self.kwargs = {}
        self.scorers = OrderedDict()
        self.refs = list(refs.parent.glob(refs.name))
        self.language = get_language(self.refs[0])
        if self.language is None:
            # Fallback to en (this is only relevant for METEOR)
            self.language = 'en'

        self.filter = None
        if filters:
            self.filter = FilterChain(filters)
            self.refs = self.filter.apply(refs)

        assert len(self.refs) > 0, "Number of reference files == 0"

        for metric in sorted(beam_metrics):
            self.kwargs[metric] = {'language': self.language}
            self.scorers[metric] = getattr(metrics, metric + 'Scorer')()

    def score(self, hyps):
        """hyps is a list of hypotheses as they come out from decoder."""
        assert isinstance(hyps, list), "hyps should be a list."

        # Post-process if requested
        if self.filter is not None:
            hyps = self.filter.apply(hyps)

        results = []
        for key, scorer in self.scorers.items():
            results.append(
                scorer.compute(self.refs, hyps, **self.kwargs[key]))
        return results

Methods

def score(self, hyps)

hyps is a list of hypotheses as they come out from decoder.

Expand source code

def score(self, hyps):
    """hyps is a list of hypotheses as they come out from decoder."""
    assert isinstance(hyps, list), "hyps should be a list."

    # Post-process if requested
    if self.filter is not None:
        hyps = self.filter.apply(hyps)

    results = []
    for key, scorer in self.scorers.items():
        results.append(
            scorer.compute(self.refs, hyps, **self.kwargs[key]))
    return results