Merge branch 'staging' into 'master'

# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.

The following applies to all files unless otherwise noted:

# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

--

This repository also contains code from Hugging Face Inc., Google Research,

and Facebook (from their Fairseq project). Files from these

organizations have notices at the top of each file. Below are licenses

used in those files, as indicated.

------------- LICENSE FOR huggingface(transformer) repository --------------

------------- LICENSE FOR huggingface and Google Research code  --------------

                                 Apache License

   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

   See the License for the specific language governing permissions and

   limitations under the License.

------------- LICENSE FOR Facebook Fairseq code --------------

MIT License

Copyright (c) Facebook, Inc. and its affiliates.

Permission is hereby granted, free of charge, to any person obtaining a copy

of this software and associated documentation files (the "Software"), to deal

in the Software without restriction, including without limitation the rights

to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

copies of the Software, and to permit persons to whom the Software is

furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all

copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

SOFTWARE.

import re

def ptb_detokenizer(string):

	string = string.replace(" '", "'")

	string = string.replace(" \n", "\n")

	string = string.replace("\n ", "\n")

	string = string.replace(" n't", "n't")

	string = string.replace(" N ","1 ")

	string = string.replace("$ 1", "$1")

	string = string.replace("# 1", "#1")

	return string

def wikitext_detokenizer(string):

	#contractions

	string = string.replace("s '", "s'")

	string = re.sub(r"/' [0-9]/", r"/'[0-9]/", string)

	# number separators

	string = string.replace(" @-@ ", "-")

	string = string.replace(" @,@ ", ",")

	string = string.replace(" @.@ ", ".")

	#punctuation

	string = string.replace(" : ", ": ")

	string = string.replace(" ; ", "; ")

	string = string.replace(" . ", ". ")

	string = string.replace(" ! ", "! ")

	string = string.replace(" ? ", "? ")

	string = string.replace(" , ", ", ")

	# double brackets

	string = re.sub(r"\(\s*([^\)]*?)\s*\)", r"(\1)", string)

	string = re.sub(r"\[\s*([^\]]*?)\s*\]", r"[\1]", string)

	string = re.sub(r"{\s*([^}]*?)\s*}", r"{\1}", string)

	string = re.sub(r"\"\s*([^\"]*?)\s*\"", r'"\1"', string)

	string = re.sub(r"'\s*([^']*?)\s*'", r"'\1'", string)

	# miscellaneous

	string = string.replace("= = = =", "====")

	string = string.replace("= = =", "===")

	string = string.replace("= =", "==")

	string = string.replace(" "+chr(176)+" ", chr(176))

	string = string.replace(" \n", "\n")

	string = string.replace("\n ", "\n")

	string = string.replace(" N ", " 1 ")

	string = string.replace(" 's", "'s")

	return string

def lambada_detokenizer(string):

	return string

def get_detokenizer(path):

	for key in DETOKENIZERS.keys():

		if key in path:

			print(key)

			return DETOKENIZERS[key]

DETOKENIZERS = {

	'ptb': ptb_detokenizer,

	'wikitext': wikitext_detokenizer,

	'lambada': lambada_detokenizer,

}

# ===========

# base images

# ===========

FROM nvcr.io/nvidia/pytorch:19.09-py3

# ===============

# system packages

# ===============

RUN apt-get update && apt-get install -y \

    bash-completion \

    emacs \

    git \

    graphviz \

    htop \

    libopenexr-dev \

    rsync \

    wget \

&& rm -rf /var/lib/apt/lists/*

# ============

# pip packages

# ============

RUN pip install --upgrade pip && \

    pip install --upgrade setuptools

COPY requirements.txt /tmp/

RUN pip install --upgrade --ignore-installed -r /tmp/requirements.txt

boto3

google-cloud-language

inflect

nltk

numpy

pandas

requests

sentencepiece

tensorflow

tqdm