1.0.0

.editorconfig

@@ -0,0 +1,16 @@
+# For more information about the properties used in
+# this file, please see the EditorConfig documentation:
+# http://editorconfig.org/
+
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.md]
+trim_trailing_whitespace = false

.eslintrc

@@ -0,0 +1,7 @@
+{
+  "extends": "airbnb/base",
+  "rules": {
+    "comma-dangle": "off",
+    "func-names": "off"
+  }
+}

.gitattributes

@@ -0,0 +1,17 @@
+# Automatically normalize line endings for all text-based files
+# http://git-scm.com/docs/gitattributes#_end_of_line_conversion
+* text=auto
+
+# For the following file types, normalize line endings to LF on
+# checkin and prevent conversion to CRLF when they are checked out
+# (this is required in order to prevent newline related issues like,
+# for example, after the build script is run)
+.*      text eol=lf
+*.css   text eol=lf
+*.html  text eol=lf
+*.js    text eol=lf
+*.json  text eol=lf
+*.md    text eol=lf
+*.sh    text eol=lf
+*.txt   text eol=lf
+*.xml   text eol=lf

.gitignore

@@ -0,0 +1 @@
+node_modules

.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: node_js
+node_js:
+  - node
+script:
+  - npm run lint
+  - npm run test

History.md

@@ -0,0 +1,3 @@
+1.0.0 - March 9, 2016
+---------------------
+* Initial release

Readme.md

@@ -0,0 +1,79 @@
+# metalsmith-robots
+
+[![npm version][version-badge]][version-url]
+[![build status][build-badge]][build-url]
+[![dependency status][dependency-badge]][dependency-url]
+[![devdependency status][devdependency-badge]][devdependency-url]
+[![downloads][downloads-badge]][downloads-url]
+
+> A metalsmith plugin for generating a robots.txt file
+
+[![stack overflow][stackoverflow-badge]][stackoverflow-url]
+[![slack chat][slack-badge]][slack-url]
+
+This plugin allows you to generate a robots.txt file. It accepts global options, and can be triggered from a file's frontmatter with the `public` and `private` keywords. Works well with [metalsmith-mapsite](https://github.com/superwolff/metalsmith-mapsite), as that also accepts setting a page to private from the frontmatter.
+
+For support questions please use [stack overflow][stackoverflow-url] or the metalsmith [slack channel][slack-url].
+
+## Installation
+
+```
+$ npm install metalsmith-robots
+```
+
+## Example
+
+Configuration in `metalsmith.json`:
+
+```json
+{
+  "plugins": {
+    "metalsmith-robots": {
+      "useragent": "googlebot",
+      "allow": ["index.html", "about.html"],
+      "disallow": ["404.html"],
+      "sitemap": "https://www.site.com/sitemap.xml"
+    }
+  }
+}
+```
+
+Which will generate the following robots.txt:
+
+```
+User-agent: googlebot
+Allow: index.html
+Allow: about.html
+Disallow: 404.html
+Sitemap: https://www.site.com/sitemap.xml
+```
+
+## Options
+
+You can pass options to `metalsmith-robots` with the [Javascript API](https://github.com/segmentio/metalsmith#api) or [CLI](https://github.com/segmentio/metalsmith#cli). The options are:
+
+* `useragent`: the useragent - String, default: `*`
+* `allow`: an array of the url(s) to allow - Array of Strings
+* `disallow`: an array of the url(s) to disallow - Array of Strings
+* `sitemap`: the sitemap url - String
+
+Besides these options, settings `public: true` or `private: true` in a file's frontmatter will add that page to the `allow` or `disallow` option respectively. `metalsmith-robots` expects at least one of the last three options, without them it will not generate a robots.txt.
+
+## License
+
+MIT
+
+[build-badge]: https://travis-ci.org/superwolff/metalsmith-robots.svg
+[build-url]: https://travis-ci.org/superwolff/metalsmith-robots
+[dependency-badge]: https://david-dm.org/superwolff/metalsmith-robots.svg
+[dependency-url]: https://david-dm.org/superwolff/metalsmith-robots
+[devdependency-badge]: https://david-dm.org/superwolff/metalsmith-robots/dev-status.svg
+[devdependency-url]: https://david-dm.org/superwolff/metalsmith-robots#info=devDependencies
+[downloads-badge]: https://img.shields.io/npm/dm/metalsmith-robots.svg
+[downloads-url]: https://www.npmjs.com/package/metalsmith-robots
+[slack-badge]: https://img.shields.io/badge/Slack-Join%20Chat%20→-blue.svg
+[slack-url]: http://metalsmith-slack.herokuapp.com/
+[stackoverflow-badge]: https://img.shields.io/badge/stack%20overflow-%23metalsmith-red.svg
+[stackoverflow-url]: http://stackoverflow.com/questions/tagged/metalsmith
+[version-badge]: https://img.shields.io/npm/v/metalsmith-robots.svg
+[version-url]: https://www.npmjs.com/package/metalsmith-robots

lib/index.js

@@ -0,0 +1,86 @@
+/* eslint-disable no-param-reassign */
+'use strict';
+
+/**
+ * Dependencies
+ */
+const debug = require('debug')('metalsmith-robots');
+const robotize = require('robotize');
+
+/**
+ * Metalsmith plugin for generating a robots.txt file.
+ *
+ * @param {Object} options
+ *   @property {String} useragent (optional)
+ *   @property {Array} allow
+ *   @property {Array} disallow
+ *   @property {String} sitemap
+ * @return {Function}
+ */
+module.exports = function plugin(opts) {
+  /**
+   * Init
+   */
+  opts = opts || {};
+
+  // Create local arrays for allowed and disallowed pages
+  const allow = [];
+  const disallow = [];
+
+  /**
+   * Main plugin function
+   */
+  return function (files, metalsmith, done) {
+    /**
+     * Check for files with `public: true` in their metadata
+     */
+    for (const file in files) {
+      if (files[file].public) {
+        debug(`file marked as public: ${file}`);
+        allow.push(file);
+      }
+    }
+
+    // Add allowed pages to options
+    if (allow.length) {
+      if (opts.allow) {
+        opts.allow = opts.allow.concat(allow);
+      } else {
+        opts.allow = allow;
+      }
+    }
+
+    /**
+     * Check for files with `private: true` in their metadata
+     */
+    for (const file in files) {
+      if (files[file].private) {
+        debug(`file marked as private: ${file}`);
+        disallow.push(file);
+      }
+    }
+
+    // Add allowed pages to options
+    if (disallow.length) {
+      if (opts.disallow) {
+        opts.disallow = opts.disallow.concat(disallow);
+      } else {
+        opts.disallow = disallow;
+      }
+    }
+
+    robotize(opts, (err, robots) => {
+      if (err) {
+        debug('skipping creation of robots.txt');
+        done();
+      } else {
+        debug('creating robots.txt');
+        // Create file
+        files['robots.txt'] = {
+          contents: new Buffer(robots)
+        };
+        done();
+      }
+    });
+  };
+};

package.json

@@ -0,0 +1,24 @@
+{
+  "name": "metalsmith-robots",
+  "author": "superwolff <[email protected]> (http://superwolff.nl/)",
+  "description": "A metalsmith plugin for generating a robots.txt file",
+  "repository": "git://github.com/superwolff/metalsmith-robots.git",
+  "version": "1.0.0",
+  "license": "MIT",
+  "main": "lib/index.js",
+  "scripts": {
+    "test": "mocha --reporter spec"
+  },
+  "dependencies": {
+    "debug": "^2.2.0",
+    "robotize": "^1.0.0"
+  },
+  "devDependencies": {
+    "assert-dir-equal": "^1.0.1",
+    "eslint": "^2.3.0",
+    "eslint-config-airbnb": "^6.1.0",
+    "eslint-plugin-react": "^4.2.1",
+    "metalsmith": "^2.1.0",
+    "mocha": "^2.4.5"
+  }
+}

test/fixtures/allow/build/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/allow/build/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Allow: allowed-1.html
+Allow: allowed-2.html

test/fixtures/allow/expected/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/allow/expected/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Allow: allowed-1.html
+Allow: allowed-2.html

test/fixtures/allow/src/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/disallow/build/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/disallow/build/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: disallowed-1.html
+Disallow: disallowed-2.html

test/fixtures/disallow/expected/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/disallow/expected/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: disallowed-1.html
+Disallow: disallowed-2.html

test/fixtures/disallow/src/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/no-empty/build/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/no-empty/expected/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/no-empty/src/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/private/build/index.html

@@ -0,0 +1 @@
+Index page

test/fixtures/private/build/nested/index.html

@@ -0,0 +1 @@
+Nested page

test/fixtures/private/build/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: index.html
+Disallow: nested/index.html

test/fixtures/private/expected/index.html

@@ -0,0 +1 @@
+Index page

test/fixtures/private/expected/nested/index.html

@@ -0,0 +1 @@
+Nested page

test/fixtures/private/expected/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: index.html
+Disallow: nested/index.html

test/fixtures/private/src/index.html

@@ -0,0 +1,4 @@
+---
+private: true
+---
+Index page

test/fixtures/private/src/nested/index.html

@@ -0,0 +1,4 @@
+---
+private: true
+---
+Nested page

test/fixtures/public/build/index.html

@@ -0,0 +1 @@
+Index page

test/fixtures/public/build/nested/index.html

@@ -0,0 +1 @@
+Nested page

test/fixtures/public/build/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Allow: index.html
+Allow: nested/index.html

test/fixtures/public/expected/index.html

@@ -0,0 +1 @@
+Index page

test/fixtures/public/expected/nested/index.html

@@ -0,0 +1 @@
+Nested page

test/fixtures/public/expected/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Allow: index.html
+Allow: nested/index.html

test/fixtures/public/src/index.html

@@ -0,0 +1,4 @@
+---
+public: true
+---
+Index page

test/fixtures/public/src/nested/index.html

@@ -0,0 +1,4 @@
+---
+public: true
+---
+Nested page

test/fixtures/sitemap/build/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/sitemap/build/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Sitemap: https://www.site.com/sitemap.xml

test/fixtures/sitemap/expected/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/sitemap/expected/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Sitemap: https://www.site.com/sitemap.xml

test/fixtures/sitemap/src/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/useragent/build/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/useragent/build/robots.txt

@@ -0,0 +1,2 @@
+User-agent: googlebot
+Sitemap: https://www.site.com/sitemap.xml

test/fixtures/useragent/expected/index.html

@@ -0,0 +1 @@
+Index.html

test/fixtures/useragent/expected/robots.txt

@@ -0,0 +1,2 @@
+User-agent: googlebot
+Sitemap: https://www.site.com/sitemap.xml

test/fixtures/useragent/src/index.html

@@ -0,0 +1 @@
+Index.html