tokens.py

Go to the documentation of this file.

#  Copyright 2008-2015 Nokia Networks
#  Copyright 2016-     Robot Framework Foundation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
 
from robot.variables import VariableIterator
 
 
 
class Token:
 
    SETTING_HEADER = 'SETTING HEADER'
    VARIABLE_HEADER = 'VARIABLE HEADER'
    TESTCASE_HEADER = 'TESTCASE HEADER'
    TASK_HEADER = 'TASK HEADER'
    KEYWORD_HEADER = 'KEYWORD HEADER'
    COMMENT_HEADER = 'COMMENT HEADER'
 
    TESTCASE_NAME = 'TESTCASE NAME'
    KEYWORD_NAME = 'KEYWORD NAME'
 
    DOCUMENTATION = 'DOCUMENTATION'
    SUITE_SETUP = 'SUITE SETUP'
    SUITE_TEARDOWN = 'SUITE TEARDOWN'
    METADATA = 'METADATA'
    TEST_SETUP = 'TEST SETUP'
    TEST_TEARDOWN = 'TEST TEARDOWN'
    TEST_TEMPLATE = 'TEST TEMPLATE'
    TEST_TIMEOUT = 'TEST TIMEOUT'
    FORCE_TAGS = 'FORCE TAGS'
    DEFAULT_TAGS = 'DEFAULT TAGS'
    KEYWORD_TAGS = 'KEYWORD TAGS'
    LIBRARY = 'LIBRARY'
    RESOURCE = 'RESOURCE'
    VARIABLES = 'VARIABLES'
    SETUP = 'SETUP'
    TEARDOWN = 'TEARDOWN'
    TEMPLATE = 'TEMPLATE'
    TIMEOUT = 'TIMEOUT'
    TAGS = 'TAGS'
    ARGUMENTS = 'ARGUMENTS'
    # Use ´RETURN_SETTING` type instead of `RETURN`. `[Return]` is deprecated and
    # `RETURN` type will be used with `RETURN` statement in the future.
    RETURN = 'RETURN'
    RETURN_SETTING = RETURN
 
    NAME = 'NAME'
    VARIABLE = 'VARIABLE'
    ARGUMENT = 'ARGUMENT'
    ASSIGN = 'ASSIGN'
    KEYWORD = 'KEYWORD'
    WITH_NAME = 'WITH NAME'
    FOR = 'FOR'
    FOR_SEPARATOR = 'FOR SEPARATOR'
    END = 'END'
    IF = 'IF'
    INLINE_IF = 'INLINE IF'
    ELSE_IF = 'ELSE IF'
    ELSE = 'ELSE'
    TRY = 'TRY'
    EXCEPT = 'EXCEPT'
    FINALLY = 'FINALLY'
    AS = 'AS'
    WHILE = 'WHILE'
    RETURN_STATEMENT = 'RETURN STATEMENT'
    CONTINUE = 'CONTINUE'
    BREAK = 'BREAK'
    OPTION = 'OPTION'
 
    SEPARATOR = 'SEPARATOR'
    COMMENT = 'COMMENT'
    CONTINUATION = 'CONTINUATION'
    CONFIG = 'CONFIG'
    EOL = 'EOL'
    EOS = 'EOS'
 
    ERROR = 'ERROR'
    FATAL_ERROR = 'FATAL ERROR'
 
    NON_DATA_TOKENS = frozenset((
        SEPARATOR,
        COMMENT,
        CONTINUATION,
        EOL,
        EOS
    ))
    SETTING_TOKENS = frozenset((
        DOCUMENTATION,
        SUITE_SETUP,
        SUITE_TEARDOWN,
        METADATA,
        TEST_SETUP,
        TEST_TEARDOWN,
        TEST_TEMPLATE,
        TEST_TIMEOUT,
        FORCE_TAGS,
        DEFAULT_TAGS,
        KEYWORD_TAGS,
        LIBRARY,
        RESOURCE,
        VARIABLES,
        SETUP,
        TEARDOWN,
        TEMPLATE,
        TIMEOUT,
        TAGS,
        ARGUMENTS,
        RETURN
    ))
    HEADER_TOKENS = frozenset((
        SETTING_HEADER,
        VARIABLE_HEADER,
        TESTCASE_HEADER,
        TASK_HEADER,
        KEYWORD_HEADER,
        COMMENT_HEADER
    ))
    ALLOW_VARIABLES = frozenset((
        NAME,
        ARGUMENT,
        TESTCASE_NAME,
        KEYWORD_NAME
    ))
 
    __slots__ = ['type', 'value', 'lineno', 'col_offset', 'error',
                 '_add_eos_before', '_add_eos_after']
 
    def __init__(self, type=None, value=None, lineno=-1, col_offset=-1, error=None):
        self.typetype = type
        if value is None:
            value = {
                Token.IF: 'IF', Token.INLINE_IF: 'IF', Token.ELSE_IF: 'ELSE IF',
                Token.ELSE: 'ELSE', Token.FOR: 'FOR', Token.WHILE: 'WHILE',
                Token.TRY: 'TRY', Token.EXCEPT: 'EXCEPT', Token.FINALLY: 'FINALLY',
                Token.END: 'END', Token.CONTINUE: 'CONTINUE', Token.BREAK: 'BREAK',
                Token.RETURN_STATEMENT: 'RETURN', Token.CONTINUATION: '...',
                Token.EOL: '\n', Token.WITH_NAME: 'WITH NAME', Token.AS: 'AS'
            }.get(type, '')
        self.valuevalue = value
        self.linenolineno = lineno
        self.col_offsetcol_offset = col_offset
        self.errorerror = error
        # Used internally be lexer to indicate that EOS is needed before/after.
        self._add_eos_before_add_eos_before = False
        self._add_eos_after_add_eos_after = False
 
    @property
    end_col_offset = property
    
    def end_col_offset(self):
        if self.col_offsetcol_offset == -1:
            return -1
        return self.col_offsetcol_offset + len(self.valuevalue)
 
    def set_error(self, error, fatal=False):
        self.typetype = Token.ERROR if not fatal else Token.FATAL_ERROR
        self.errorerror = error
 
    
    def tokenize_variables(self):
        if self.typetype not in Token.ALLOW_VARIABLES:
            return self._tokenize_no_variables_tokenize_no_variables()
        variables = VariableIterator(self.valuevalue)
        if not variables:
            return self._tokenize_no_variables_tokenize_no_variables()
        return self._tokenize_variables_tokenize_variables(variables)
 
    def _tokenize_no_variables(self):
        yield self
 
    def _tokenize_variables(self, variables):
        lineno = self.linenolineno
        col_offset = self.col_offsetcol_offset
        remaining = ''
        for before, variable, remaining in variables:
            if before:
                yield Token(self.typetype, before, lineno, col_offset)
                col_offset += len(before)
            yield Token(Token.VARIABLE, variable, lineno, col_offset)
            col_offset += len(variable)
        if remaining:
            yield Token(self.typetype, remaining, lineno, col_offset)
 
    def __str__(self):
        return self.valuevalue
 
    def __repr__(self):
        type_ = self.typetype.replace(' ', '_') if self.typetype else 'None'
        error = '' if not self.errorerror else ', %r' % self.errorerror
        return 'Token(%s, %r, %s, %s%s)' % (type_, self.valuevalue, self.linenolineno,
                                            self.col_offsetcol_offset, error)
 
    def __eq__(self, other):
        return (isinstance(other, Token)
                and self.typetype == other.type
                and self.valuevalue == other.value
                and self.linenolineno == other.lineno
                and self.col_offsetcol_offset == other.col_offset
                and self.errorerror == other.error)
 
 
 
class EOS(Token):
    __slots__ = []
 
    def __init__(self, lineno=-1, col_offset=-1):
        Token.__init__(self, Token.EOS, '', lineno, col_offset)
 
    @classmethod
    def from_token(cls, token, before=False):
        col_offset = token.col_offset if before else token.end_col_offset
        return EOS(token.lineno, col_offset)
 
 
 
class END(Token):
    __slots__ = []
 
    def __init__(self, lineno=-1, col_offset=-1, virtual=False):
        value = 'END' if not virtual else ''
        Token.__init__(self, Token.END, value, lineno, col_offset)
 
    @classmethod
    def from_token(cls, token, virtual=False):
        return END(token.lineno, token.end_col_offset, virtual)