samples-index.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2014 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Build wiki page with a list of all samples.

The information for the wiki page is built from data found in all the README
files in the samples. The format of the README file is:


   Description is everything up to the first blank line.

   api: plus  (Used to look up the long name in discovery).
   keywords: appengine (such as appengine, oauth2, cmdline)

   The rest of the file is ignored when it comes to building the index.
"""
from __future__ import print_function

import httplib2
import itertools
import json
import os
import re

BASE_HG_URI = "http://code.google.com/p/google-api-python-client/source/" "browse/#hg"

http = httplib2.Http(".cache")
r, c = http.request("https://www.googleapis.com/discovery/v1/apis")
if r.status != 200:
    raise ValueError("Received non-200 response when retrieving Discovery.")

# Dictionary mapping api names to their discovery description.
DIRECTORY = {}
for item in json.loads(c)["items"]:
    if item["preferred"]:
        DIRECTORY[item["name"]] = item

# A list of valid keywords. Should not be taken as complete, add to
# this list as needed.
KEYWORDS = {
    "appengine": "Google App Engine",
    "oauth2": "OAuth 2.0",
    "cmdline": "Command-line",
    "django": "Django",
    "threading": "Threading",
    "pagination": "Pagination",
    "media": "Media Upload and Download",
}


def get_lines(name, lines):
    """Return lines that begin with name.

  Lines are expected to look like:

     name: space separated values

  Args:
    name: string, parameter name.
    lines: iterable of string, lines in the file.

  Returns:
    List of values in the lines that match.
  """
    retval = []
    matches = itertools.ifilter(lambda x: x.startswith(name + ":"), lines)
    for line in matches:
        retval.extend(line[len(name) + 1 :].split())
    return retval


def wiki_escape(s):
    """Detect WikiSyntax (i.e. InterCaps, a.k.a. CamelCase) and escape it."""
    ret = []
    for word in s.split():
        if re.match(r"[A-Z]+[a-z]+[A-Z]", word):
            word = "!%s" % word
        ret.append(word)
    return " ".join(ret)


def context_from_sample(api, keywords, dirname, desc, uri):
    """Return info for expanding a sample into a template.

  Args:
    api: string, name of api.
    keywords: list of string, list of keywords for the given api.
    dirname: string, directory name of the sample.
    desc: string, long description of the sample.
    uri: string, uri of the sample code if provided in the README.

  Returns:
    A dictionary of values useful for template expansion.
  """
    if uri is None:
        uri = BASE_HG_URI + dirname.replace("/", "%2F")
    else:
        uri = "".join(uri)
    if api is None:
        return None
    else:
        entry = DIRECTORY[api]
        context = {
            "api": api,
            "version": entry["version"],
            "api_name": wiki_escape(entry.get("title", entry.get("description"))),
            "api_desc": wiki_escape(entry["description"]),
            "api_icon": entry["icons"]["x32"],
            "keywords": keywords,
            "dir": dirname,
            "uri": uri,
            "desc": wiki_escape(desc),
        }
        return context


def keyword_context_from_sample(keywords, dirname, desc, uri):
    """Return info for expanding a sample into a template.

  Sample may not be about a specific api.

  Args:
    keywords: list of string, list of keywords for the given api.
    dirname: string, directory name of the sample.
    desc: string, long description of the sample.
    uri: string, uri of the sample code if provided in the README.

  Returns:
    A dictionary of values useful for template expansion.
  """
    if uri is None:
        uri = BASE_HG_URI + dirname.replace("/", "%2F")
    else:
        uri = "".join(uri)
    context = {
        "keywords": keywords,
        "dir": dirname,
        "uri": uri,
        "desc": wiki_escape(desc),
    }
    return context


def scan_readme_files(dirname):
    """Scans all subdirs of dirname for README files.

  Args:
    dirname: string, name of directory to walk.

  Returns:
    (samples, keyword_set): list of information about all samples, the union
      of all keywords found.
  """
    samples = []
    keyword_set = set()

    for root, dirs, files in os.walk(dirname):
        if "README" in files:
            filename = os.path.join(root, "README")
            with open(filename, "r") as f:
                content = f.read()
                lines = content.splitlines()
                desc = " ".join(itertools.takewhile(lambda x: x, lines))
                api = get_lines("api", lines)
                keywords = get_lines("keywords", lines)
                uri = get_lines("uri", lines)
                if not uri:
                    uri = None

                for k in keywords:
                    if k not in KEYWORDS:
                        raise ValueError(
                            "%s is not a valid keyword in file %s" % (k, filename)
                        )
                keyword_set.update(keywords)
                if not api:
                    api = [None]
                samples.append((api[0], keywords, root[1:], desc, uri))

    samples.sort()

    return samples, keyword_set


def main():
    # Get all the information we need out of the README files in the samples.
    samples, keyword_set = scan_readme_files("./samples")

    # Now build a wiki page with all that information. Accumulate all the
    # information as string to be concatenated when were done.
    page = ['<wiki:toc max_depth="3" />\n= Samples By API =\n']

    # All the samples, grouped by API.
    current_api = None
    for api, keywords, dirname, desc, uri in samples:
        context = context_from_sample(api, keywords, dirname, desc, uri)
        if context is None:
            continue
        if current_api != api:
            page.append(
                """
=== %(api_icon)s %(api_name)s ===

%(api_desc)s

Documentation for the %(api_name)s in [https://google-api-client-libraries.appspot.com/documentation/%(api)s/%(version)s/python/latest/ PyDoc]

"""
                % context
            )
            current_api = api

        page.append("|| [%(uri)s %(dir)s] || %(desc)s ||\n" % context)

    # Now group the samples by keywords.
    for keyword, keyword_name in KEYWORDS.iteritems():
        if keyword not in keyword_set:
            continue
        page.append("\n= %s Samples =\n\n" % keyword_name)
        page.append("<table border=1 cellspacing=0 cellpadding=8px>\n")
        for _, keywords, dirname, desc, uri in samples:
            context = keyword_context_from_sample(keywords, dirname, desc, uri)
            if keyword not in keywords:
                continue
            page.append(
                """
<tr>
  <td>[%(uri)s %(dir)s] </td>
  <td> %(desc)s </td>
</tr>"""
                % context
            )
        page.append("</table>\n")

    print("".join(page))


if __name__ == "__main__":
    main()