From 3a2ed920e67bbd62c5e85e2d1701d64b2af15c83 Mon Sep 17 00:00:00 2001
From: Jonathan Hsieh <jon@cloudera.com>
Date: Sat, 30 Apr 2011 23:32:28 -0700
Subject: [PATCH] FLUME-630: Maven builds breaks doc build

---
 docs/Cookbook/breadcrumbs.xsl                      |   47 -
 docs/Cookbook/docbook.css                          |  269 --
 docs/Cookbook/footer.xsl                           |   47 -
 docs/Cookbook/header.xsl                           |   41 -
 docs/Cookbook/html.xsl                             |   84 -
 docs/Cookbook/images/README                        |    5 -
 docs/Cookbook/images/callouts/1.png                |  Bin 329 -> 0 bytes
 docs/Cookbook/images/callouts/10.png               |  Bin 361 -> 0 bytes
 docs/Cookbook/images/callouts/11.png               |  Bin 565 -> 0 bytes
 docs/Cookbook/images/callouts/12.png               |  Bin 617 -> 0 bytes
 docs/Cookbook/images/callouts/13.png               |  Bin 623 -> 0 bytes
 docs/Cookbook/images/callouts/14.png               |  Bin 411 -> 0 bytes
 docs/Cookbook/images/callouts/15.png               |  Bin 640 -> 0 bytes
 docs/Cookbook/images/callouts/2.png                |  Bin 353 -> 0 bytes
 docs/Cookbook/images/callouts/3.png                |  Bin 350 -> 0 bytes
 docs/Cookbook/images/callouts/4.png                |  Bin 345 -> 0 bytes
 docs/Cookbook/images/callouts/5.png                |  Bin 348 -> 0 bytes
 docs/Cookbook/images/callouts/6.png                |  Bin 355 -> 0 bytes
 docs/Cookbook/images/callouts/7.png                |  Bin 344 -> 0 bytes
 docs/Cookbook/images/callouts/8.png                |  Bin 357 -> 0 bytes
 docs/Cookbook/images/callouts/9.png                |  Bin 357 -> 0 bytes
 docs/Cookbook/images/caution.png                   |  Bin 2554 -> 0 bytes
 docs/Cookbook/images/example.png                   |  Bin 2354 -> 0 bytes
 docs/Cookbook/images/home.png                      |  Bin 1340 -> 0 bytes
 docs/Cookbook/images/important.png                 |  Bin 2657 -> 0 bytes
 docs/Cookbook/images/next.png                      |  Bin 1302 -> 0 bytes
 docs/Cookbook/images/note.png                      |  Bin 2730 -> 0 bytes
 docs/Cookbook/images/prev.png                      |  Bin 1348 -> 0 bytes
 docs/Cookbook/images/tip.png                       |  Bin 2602 -> 0 bytes
 docs/Cookbook/images/up.png                        |  Bin 1320 -> 0 bytes
 docs/Cookbook/images/warning.png                   |  Bin 2828 -> 0 bytes
 docs/Cookbook/index.html                           |  289 --
 docs/Cookbook/index.txt                            |   27 -
 docs/Cookbook/index.xml                            |  493 ----
 docs/DeveloperGuide/breadcrumbs.xsl                |   47 -
 docs/DeveloperGuide/docbook.css                    |  269 --
 docs/DeveloperGuide/footer.xsl                     |   47 -
 docs/DeveloperGuide/header.xsl                     |   41 -
 docs/DeveloperGuide/html.xsl                       |   84 -
 docs/DeveloperGuide/images/README                  |    5 -
 docs/DeveloperGuide/images/callouts/1.png          |  Bin 329 -> 0 bytes
 docs/DeveloperGuide/images/callouts/10.png         |  Bin 361 -> 0 bytes
 docs/DeveloperGuide/images/callouts/11.png         |  Bin 565 -> 0 bytes
 docs/DeveloperGuide/images/callouts/12.png         |  Bin 617 -> 0 bytes
 docs/DeveloperGuide/images/callouts/13.png         |  Bin 623 -> 0 bytes
 docs/DeveloperGuide/images/callouts/14.png         |  Bin 411 -> 0 bytes
 docs/DeveloperGuide/images/callouts/15.png         |  Bin 640 -> 0 bytes
 docs/DeveloperGuide/images/callouts/2.png          |  Bin 353 -> 0 bytes
 docs/DeveloperGuide/images/callouts/3.png          |  Bin 350 -> 0 bytes
 docs/DeveloperGuide/images/callouts/4.png          |  Bin 345 -> 0 bytes
 docs/DeveloperGuide/images/callouts/5.png          |  Bin 348 -> 0 bytes
 docs/DeveloperGuide/images/callouts/6.png          |  Bin 355 -> 0 bytes
 docs/DeveloperGuide/images/callouts/7.png          |  Bin 344 -> 0 bytes
 docs/DeveloperGuide/images/callouts/8.png          |  Bin 357 -> 0 bytes
 docs/DeveloperGuide/images/callouts/9.png          |  Bin 357 -> 0 bytes
 docs/DeveloperGuide/images/caution.png             |  Bin 2554 -> 0 bytes
 docs/DeveloperGuide/images/example.png             |  Bin 2354 -> 0 bytes
 docs/DeveloperGuide/images/home.png                |  Bin 1340 -> 0 bytes
 docs/DeveloperGuide/images/important.png           |  Bin 2657 -> 0 bytes
 docs/DeveloperGuide/images/next.png                |  Bin 1302 -> 0 bytes
 docs/DeveloperGuide/images/note.png                |  Bin 2730 -> 0 bytes
 docs/DeveloperGuide/images/prev.png                |  Bin 1348 -> 0 bytes
 docs/DeveloperGuide/images/tip.png                 |  Bin 2602 -> 0 bytes
 docs/DeveloperGuide/images/up.png                  |  Bin 1320 -> 0 bytes
 docs/DeveloperGuide/images/warning.png             |  Bin 2828 -> 0 bytes
 docs/DeveloperGuide/index.html                     |   28 -
 docs/DeveloperGuide/index.txt                      |   27 -
 docs/UserGuide/architecture.png                    |  Bin 41884 -> 0 bytes
 docs/UserGuide/breadcrumbs.xsl                     |   47 -
 docs/UserGuide/docbook.css                         |  269 --
 docs/UserGuide/failoverCollector.png               |  Bin 34176 -> 0 bytes
 docs/UserGuide/footer.xsl                          |   47 -
 docs/UserGuide/header.xsl                          |   41 -
 docs/UserGuide/html.xsl                            |   84 -
 docs/UserGuide/images/README                       |    5 -
 docs/UserGuide/images/callouts/1.png               |  Bin 329 -> 0 bytes
 docs/UserGuide/images/callouts/10.png              |  Bin 361 -> 0 bytes
 docs/UserGuide/images/callouts/11.png              |  Bin 565 -> 0 bytes
 docs/UserGuide/images/callouts/12.png              |  Bin 617 -> 0 bytes
 docs/UserGuide/images/callouts/13.png              |  Bin 623 -> 0 bytes
 docs/UserGuide/images/callouts/14.png              |  Bin 411 -> 0 bytes
 docs/UserGuide/images/callouts/15.png              |  Bin 640 -> 0 bytes
 docs/UserGuide/images/callouts/2.png               |  Bin 353 -> 0 bytes
 docs/UserGuide/images/callouts/3.png               |  Bin 350 -> 0 bytes
 docs/UserGuide/images/callouts/4.png               |  Bin 345 -> 0 bytes
 docs/UserGuide/images/callouts/5.png               |  Bin 348 -> 0 bytes
 docs/UserGuide/images/callouts/6.png               |  Bin 355 -> 0 bytes
 docs/UserGuide/images/callouts/7.png               |  Bin 344 -> 0 bytes
 docs/UserGuide/images/callouts/8.png               |  Bin 357 -> 0 bytes
 docs/UserGuide/images/callouts/9.png               |  Bin 357 -> 0 bytes
 docs/UserGuide/images/caution.png                  |  Bin 2554 -> 0 bytes
 docs/UserGuide/images/example.png                  |  Bin 2354 -> 0 bytes
 docs/UserGuide/images/home.png                     |  Bin 1340 -> 0 bytes
 docs/UserGuide/images/important.png                |  Bin 2657 -> 0 bytes
 docs/UserGuide/images/next.png                     |  Bin 1302 -> 0 bytes
 docs/UserGuide/images/note.png                     |  Bin 2730 -> 0 bytes
 docs/UserGuide/images/prev.png                     |  Bin 1348 -> 0 bytes
 docs/UserGuide/images/tip.png                      |  Bin 2602 -> 0 bytes
 docs/UserGuide/images/up.png                       |  Bin 1320 -> 0 bytes
 docs/UserGuide/images/warning.png                  |  Bin 2828 -> 0 bytes
 docs/UserGuide/index.html                          | 3113 --------------------
 docs/UserGuide/index.txt                           |   27 -
 docs/UserGuide/master-zk-internal.png              |  Bin 62376 -> 0 bytes
 docs/UserGuide/master-zk-standalone.png            |  Bin 43626 -> 0 bytes
 docs/UserGuide/multi-flow.png                      |  Bin 35556 -> 0 bytes
 docs/UserGuide/multiCollector.png                  |  Bin 26806 -> 0 bytes
 docs/UserGuide/single-flow.png                     |  Bin 38734 -> 0 bytes
 docs/UserGuide/singleCollector.png                 |  Bin 25527 -> 0 bytes
 docs/UserGuide/sinkStatesAppending.png             |  Bin 48600 -> 0 bytes
 docs/UserGuide/sinkStatesBuffered.png              |  Bin 46019 -> 0 bytes
 docs/UserGuide/sinkStatesOpening.png               |  Bin 45843 -> 0 bytes
 docs/UserGuide/sinkStatesSimple.png                |  Bin 24235 -> 0 bytes
 docs/UserGuide/sourceStatesBuffered.png            |  Bin 42341 -> 0 bytes
 docs/UserGuide/sourceStatesSimple.png              |  Bin 23471 -> 0 bytes
 docs/UserGuide/tiers-console.png                   |  Bin 6164 -> 0 bytes
 docs/UserGuide/tiers-hdfs.png                      |  Bin 7208 -> 0 bytes
 docs/breadcrumbs.xsl                               |   47 -
 docs/docbook.css                                   |  269 --
 docs/footer.xsl                                    |   47 -
 docs/header.xsl                                    |   41 -
 docs/html.xsl                                      |   84 -
 docs/images/README                                 |    5 -
 docs/images/callouts/1.png                         |  Bin 329 -> 0 bytes
 docs/images/callouts/10.png                        |  Bin 361 -> 0 bytes
 docs/images/callouts/11.png                        |  Bin 565 -> 0 bytes
 docs/images/callouts/12.png                        |  Bin 617 -> 0 bytes
 docs/images/callouts/13.png                        |  Bin 623 -> 0 bytes
 docs/images/callouts/14.png                        |  Bin 411 -> 0 bytes
 docs/images/callouts/15.png                        |  Bin 640 -> 0 bytes
 docs/images/callouts/2.png                         |  Bin 353 -> 0 bytes
 docs/images/callouts/3.png                         |  Bin 350 -> 0 bytes
 docs/images/callouts/4.png                         |  Bin 345 -> 0 bytes
 docs/images/callouts/5.png                         |  Bin 348 -> 0 bytes
 docs/images/callouts/6.png                         |  Bin 355 -> 0 bytes
 docs/images/callouts/7.png                         |  Bin 344 -> 0 bytes
 docs/images/callouts/8.png                         |  Bin 357 -> 0 bytes
 docs/images/callouts/9.png                         |  Bin 357 -> 0 bytes
 docs/images/caution.png                            |  Bin 2554 -> 0 bytes
 docs/images/example.png                            |  Bin 2354 -> 0 bytes
 docs/images/home.png                               |  Bin 1340 -> 0 bytes
 docs/images/important.png                          |  Bin 2657 -> 0 bytes
 docs/images/next.png                               |  Bin 1302 -> 0 bytes
 docs/images/note.png                               |  Bin 2730 -> 0 bytes
 docs/images/prev.png                               |  Bin 1348 -> 0 bytes
 docs/images/tip.png                                |  Bin 2602 -> 0 bytes
 docs/images/up.png                                 |  Bin 1320 -> 0 bytes
 docs/images/warning.png                            |  Bin 2828 -> 0 bytes
 docs/index.html                                    |   19 -
 docs/index.txt                                     |   27 -
 flume-distribution/src/main/assembly/dist.xml      |    9 +-
 flume-docs/build.xml                               |  198 ++
 flume-docs/pom.xml                                 |   42 +
 flume-docs/src/docs/DeveloperGuide/FullGuide       |   38 +
 flume-docs/src/docs/DeveloperGuide/Logging         |   51 +
 flume-docs/src/docs/DeveloperGuide/RunningTests    |   42 +
 .../src/docs/DeveloperGuide/WritingDocumentation   |   22 +
 flume-docs/src/docs/UserGuide/Advanced             |  367 +++
 flume-docs/src/docs/UserGuide/Appendix             |  466 +++
 flume-docs/src/docs/UserGuide/CustomInput          |  185 ++
 flume-docs/src/docs/UserGuide/DefaultPorts         |   48 +
 flume-docs/src/docs/UserGuide/Environment          |   59 +
 flume-docs/src/docs/UserGuide/FlumeShell           |  208 ++
 flume-docs/src/docs/UserGuide/FullGuide            |   47 +
 flume-docs/src/docs/UserGuide/FullyDist            |  711 +++++
 flume-docs/src/docs/UserGuide/Glossary             |   44 +
 flume-docs/src/docs/UserGuide/Installation         |  192 ++
 flume-docs/src/docs/UserGuide/Introduction         |  380 +++
 flume-docs/src/docs/UserGuide/LogicalNodes         |  357 +++
 flume-docs/src/docs/UserGuide/Plugins              |  607 ++++
 flume-docs/src/docs/UserGuide/PseudoDist           |  471 +++
 flume-docs/src/docs/UserGuide/QuickStart           |  292 ++
 flume-docs/src/docs/UserGuide/RateLimit            |   75 +
 flume-docs/src/docs/UserGuide/Security             |  260 ++
 flume-docs/src/docs/UserGuide/Troubleshooting      |  130 +
 flume-docs/src/docs/UserGuide/UsingOutput          |  310 ++
 flume-docs/src/docs/UserGuide/VersionHistory       |   65 +
 flume-docs/src/docs/Windows/Windows                |  208 ++
 flume-docs/src/docs/Windows/flume-logo.png         |  Bin 0 -> 32773 bytes
 .../src/docs/Windows/flume-setup-2-license.png     |  Bin 0 -> 24148 bytes
 .../src/docs/Windows/flume-setup-3-components.png  |  Bin 0 -> 22911 bytes
 .../src/docs/Windows/flume-setup-4-master.png      |  Bin 0 -> 18315 bytes
 flume-docs/src/docs/Windows/flume-setup-5-jre.png  |  Bin 0 -> 21042 bytes
 .../src/docs/Windows/flume-setup-6-installdir.png  |  Bin 0 -> 21547 bytes
 .../src/docs/Windows/flume-setup-7-complete.png    |  Bin 0 -> 18194 bytes
 flume-docs/src/docs/Windows/flume-setup.png        |  Bin 0 -> 18506 bytes
 flume-docs/src/docs/Windows/flume-tray-icon.png    |  Bin 0 -> 3080 bytes
 flume-docs/src/docs/Windows/flume-tray-menu.png    |  Bin 0 -> 3389 bytes
 flume-docs/src/docs/cookbook/Apache                |  178 ++
 flume-docs/src/docs/cookbook/Cookbook              |   65 +
 flume-docs/src/docs/cookbook/Scribe                |   60 +
 flume-docs/src/docs/cookbook/Syslog                |  245 ++
 flume-docs/src/docs/cookbook/Testing               |  173 ++
 flume-docs/src/docs/web/breadcrumbs.xsl            |   47 +
 flume-docs/src/docs/web/docbook.css                |  269 ++
 flume-docs/src/docs/web/footer.xsl                 |   47 +
 flume-docs/src/docs/web/header.xsl                 |   41 +
 flume-docs/src/docs/web/html.xsl                   |   84 +
 flume-docs/src/docs/web/images/README              |    5 +
 flume-docs/src/docs/web/images/callouts/1.png      |  Bin 0 -> 329 bytes
 flume-docs/src/docs/web/images/callouts/10.png     |  Bin 0 -> 361 bytes
 flume-docs/src/docs/web/images/callouts/11.png     |  Bin 0 -> 565 bytes
 flume-docs/src/docs/web/images/callouts/12.png     |  Bin 0 -> 617 bytes
 flume-docs/src/docs/web/images/callouts/13.png     |  Bin 0 -> 623 bytes
 flume-docs/src/docs/web/images/callouts/14.png     |  Bin 0 -> 411 bytes
 flume-docs/src/docs/web/images/callouts/15.png     |  Bin 0 -> 640 bytes
 flume-docs/src/docs/web/images/callouts/2.png      |  Bin 0 -> 353 bytes
 flume-docs/src/docs/web/images/callouts/3.png      |  Bin 0 -> 350 bytes
 flume-docs/src/docs/web/images/callouts/4.png      |  Bin 0 -> 345 bytes
 flume-docs/src/docs/web/images/callouts/5.png      |  Bin 0 -> 348 bytes
 flume-docs/src/docs/web/images/callouts/6.png      |  Bin 0 -> 355 bytes
 flume-docs/src/docs/web/images/callouts/7.png      |  Bin 0 -> 344 bytes
 flume-docs/src/docs/web/images/callouts/8.png      |  Bin 0 -> 357 bytes
 flume-docs/src/docs/web/images/callouts/9.png      |  Bin 0 -> 357 bytes
 flume-docs/src/docs/web/images/caution.png         |  Bin 0 -> 2554 bytes
 flume-docs/src/docs/web/images/example.png         |  Bin 0 -> 2354 bytes
 flume-docs/src/docs/web/images/home.png            |  Bin 0 -> 1340 bytes
 flume-docs/src/docs/web/images/important.png       |  Bin 0 -> 2657 bytes
 flume-docs/src/docs/web/images/next.png            |  Bin 0 -> 1302 bytes
 flume-docs/src/docs/web/images/note.png            |  Bin 0 -> 2730 bytes
 flume-docs/src/docs/web/images/prev.png            |  Bin 0 -> 1348 bytes
 flume-docs/src/docs/web/images/tip.png             |  Bin 0 -> 2602 bytes
 flume-docs/src/docs/web/images/up.png              |  Bin 0 -> 1320 bytes
 flume-docs/src/docs/web/images/warning.png         |  Bin 0 -> 2828 bytes
 flume-docs/src/docs/web/index.txt                  |   27 +
 pom.xml                                            |    1 +
 225 files changed, 7125 insertions(+), 6023 deletions(-)
 delete mode 100644 docs/Cookbook/breadcrumbs.xsl
 delete mode 100644 docs/Cookbook/docbook.css
 delete mode 100644 docs/Cookbook/footer.xsl
 delete mode 100644 docs/Cookbook/header.xsl
 delete mode 100644 docs/Cookbook/html.xsl
 delete mode 100644 docs/Cookbook/images/README
 delete mode 100644 docs/Cookbook/images/callouts/1.png
 delete mode 100644 docs/Cookbook/images/callouts/10.png
 delete mode 100644 docs/Cookbook/images/callouts/11.png
 delete mode 100644 docs/Cookbook/images/callouts/12.png
 delete mode 100644 docs/Cookbook/images/callouts/13.png
 delete mode 100644 docs/Cookbook/images/callouts/14.png
 delete mode 100644 docs/Cookbook/images/callouts/15.png
 delete mode 100644 docs/Cookbook/images/callouts/2.png
 delete mode 100644 docs/Cookbook/images/callouts/3.png
 delete mode 100644 docs/Cookbook/images/callouts/4.png
 delete mode 100644 docs/Cookbook/images/callouts/5.png
 delete mode 100644 docs/Cookbook/images/callouts/6.png
 delete mode 100644 docs/Cookbook/images/callouts/7.png
 delete mode 100644 docs/Cookbook/images/callouts/8.png
 delete mode 100644 docs/Cookbook/images/callouts/9.png
 delete mode 100644 docs/Cookbook/images/caution.png
 delete mode 100644 docs/Cookbook/images/example.png
 delete mode 100644 docs/Cookbook/images/home.png
 delete mode 100644 docs/Cookbook/images/important.png
 delete mode 100644 docs/Cookbook/images/next.png
 delete mode 100644 docs/Cookbook/images/note.png
 delete mode 100644 docs/Cookbook/images/prev.png
 delete mode 100644 docs/Cookbook/images/tip.png
 delete mode 100644 docs/Cookbook/images/up.png
 delete mode 100644 docs/Cookbook/images/warning.png
 delete mode 100644 docs/Cookbook/index.html
 delete mode 100644 docs/Cookbook/index.txt
 delete mode 100644 docs/Cookbook/index.xml
 delete mode 100644 docs/DeveloperGuide/breadcrumbs.xsl
 delete mode 100644 docs/DeveloperGuide/docbook.css
 delete mode 100644 docs/DeveloperGuide/footer.xsl
 delete mode 100644 docs/DeveloperGuide/header.xsl
 delete mode 100644 docs/DeveloperGuide/html.xsl
 delete mode 100644 docs/DeveloperGuide/images/README
 delete mode 100644 docs/DeveloperGuide/images/callouts/1.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/10.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/11.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/12.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/13.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/14.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/15.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/2.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/3.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/4.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/5.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/6.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/7.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/8.png
 delete mode 100644 docs/DeveloperGuide/images/callouts/9.png
 delete mode 100644 docs/DeveloperGuide/images/caution.png
 delete mode 100644 docs/DeveloperGuide/images/example.png
 delete mode 100644 docs/DeveloperGuide/images/home.png
 delete mode 100644 docs/DeveloperGuide/images/important.png
 delete mode 100644 docs/DeveloperGuide/images/next.png
 delete mode 100644 docs/DeveloperGuide/images/note.png
 delete mode 100644 docs/DeveloperGuide/images/prev.png
 delete mode 100644 docs/DeveloperGuide/images/tip.png
 delete mode 100644 docs/DeveloperGuide/images/up.png
 delete mode 100644 docs/DeveloperGuide/images/warning.png
 delete mode 100644 docs/DeveloperGuide/index.html
 delete mode 100644 docs/DeveloperGuide/index.txt
 delete mode 100644 docs/UserGuide/architecture.png
 delete mode 100644 docs/UserGuide/breadcrumbs.xsl
 delete mode 100644 docs/UserGuide/docbook.css
 delete mode 100644 docs/UserGuide/failoverCollector.png
 delete mode 100644 docs/UserGuide/footer.xsl
 delete mode 100644 docs/UserGuide/header.xsl
 delete mode 100644 docs/UserGuide/html.xsl
 delete mode 100644 docs/UserGuide/images/README
 delete mode 100644 docs/UserGuide/images/callouts/1.png
 delete mode 100644 docs/UserGuide/images/callouts/10.png
 delete mode 100644 docs/UserGuide/images/callouts/11.png
 delete mode 100644 docs/UserGuide/images/callouts/12.png
 delete mode 100644 docs/UserGuide/images/callouts/13.png
 delete mode 100644 docs/UserGuide/images/callouts/14.png
 delete mode 100644 docs/UserGuide/images/callouts/15.png
 delete mode 100644 docs/UserGuide/images/callouts/2.png
 delete mode 100644 docs/UserGuide/images/callouts/3.png
 delete mode 100644 docs/UserGuide/images/callouts/4.png
 delete mode 100644 docs/UserGuide/images/callouts/5.png
 delete mode 100644 docs/UserGuide/images/callouts/6.png
 delete mode 100644 docs/UserGuide/images/callouts/7.png
 delete mode 100644 docs/UserGuide/images/callouts/8.png
 delete mode 100644 docs/UserGuide/images/callouts/9.png
 delete mode 100644 docs/UserGuide/images/caution.png
 delete mode 100644 docs/UserGuide/images/example.png
 delete mode 100644 docs/UserGuide/images/home.png
 delete mode 100644 docs/UserGuide/images/important.png
 delete mode 100644 docs/UserGuide/images/next.png
 delete mode 100644 docs/UserGuide/images/note.png
 delete mode 100644 docs/UserGuide/images/prev.png
 delete mode 100644 docs/UserGuide/images/tip.png
 delete mode 100644 docs/UserGuide/images/up.png
 delete mode 100644 docs/UserGuide/images/warning.png
 delete mode 100644 docs/UserGuide/index.html
 delete mode 100644 docs/UserGuide/index.txt
 delete mode 100644 docs/UserGuide/master-zk-internal.png
 delete mode 100644 docs/UserGuide/master-zk-standalone.png
 delete mode 100644 docs/UserGuide/multi-flow.png
 delete mode 100644 docs/UserGuide/multiCollector.png
 delete mode 100644 docs/UserGuide/single-flow.png
 delete mode 100644 docs/UserGuide/singleCollector.png
 delete mode 100644 docs/UserGuide/sinkStatesAppending.png
 delete mode 100644 docs/UserGuide/sinkStatesBuffered.png
 delete mode 100644 docs/UserGuide/sinkStatesOpening.png
 delete mode 100644 docs/UserGuide/sinkStatesSimple.png
 delete mode 100644 docs/UserGuide/sourceStatesBuffered.png
 delete mode 100644 docs/UserGuide/sourceStatesSimple.png
 delete mode 100644 docs/UserGuide/tiers-console.png
 delete mode 100644 docs/UserGuide/tiers-hdfs.png
 delete mode 100644 docs/breadcrumbs.xsl
 delete mode 100644 docs/docbook.css
 delete mode 100644 docs/footer.xsl
 delete mode 100644 docs/header.xsl
 delete mode 100644 docs/html.xsl
 delete mode 100644 docs/images/README
 delete mode 100644 docs/images/callouts/1.png
 delete mode 100644 docs/images/callouts/10.png
 delete mode 100644 docs/images/callouts/11.png
 delete mode 100644 docs/images/callouts/12.png
 delete mode 100644 docs/images/callouts/13.png
 delete mode 100644 docs/images/callouts/14.png
 delete mode 100644 docs/images/callouts/15.png
 delete mode 100644 docs/images/callouts/2.png
 delete mode 100644 docs/images/callouts/3.png
 delete mode 100644 docs/images/callouts/4.png
 delete mode 100644 docs/images/callouts/5.png
 delete mode 100644 docs/images/callouts/6.png
 delete mode 100644 docs/images/callouts/7.png
 delete mode 100644 docs/images/callouts/8.png
 delete mode 100644 docs/images/callouts/9.png
 delete mode 100644 docs/images/caution.png
 delete mode 100644 docs/images/example.png
 delete mode 100644 docs/images/home.png
 delete mode 100644 docs/images/important.png
 delete mode 100644 docs/images/next.png
 delete mode 100644 docs/images/note.png
 delete mode 100644 docs/images/prev.png
 delete mode 100644 docs/images/tip.png
 delete mode 100644 docs/images/up.png
 delete mode 100644 docs/images/warning.png
 delete mode 100644 docs/index.html
 delete mode 100644 docs/index.txt
 create mode 100644 flume-docs/build.xml
 create mode 100644 flume-docs/pom.xml
 create mode 100644 flume-docs/src/docs/DeveloperGuide/FullGuide
 create mode 100644 flume-docs/src/docs/DeveloperGuide/Logging
 create mode 100644 flume-docs/src/docs/DeveloperGuide/RunningTests
 create mode 100644 flume-docs/src/docs/DeveloperGuide/WritingDocumentation
 create mode 100644 flume-docs/src/docs/UserGuide/Advanced
 create mode 100644 flume-docs/src/docs/UserGuide/Appendix
 create mode 100644 flume-docs/src/docs/UserGuide/CustomInput
 create mode 100644 flume-docs/src/docs/UserGuide/DefaultPorts
 create mode 100644 flume-docs/src/docs/UserGuide/Environment
 create mode 100644 flume-docs/src/docs/UserGuide/FlumeShell
 create mode 100644 flume-docs/src/docs/UserGuide/FullGuide
 create mode 100644 flume-docs/src/docs/UserGuide/FullyDist
 create mode 100644 flume-docs/src/docs/UserGuide/Glossary
 create mode 100644 flume-docs/src/docs/UserGuide/Installation
 create mode 100644 flume-docs/src/docs/UserGuide/Introduction
 create mode 100644 flume-docs/src/docs/UserGuide/LogicalNodes
 create mode 100644 flume-docs/src/docs/UserGuide/Plugins
 create mode 100644 flume-docs/src/docs/UserGuide/PseudoDist
 create mode 100644 flume-docs/src/docs/UserGuide/QuickStart
 create mode 100644 flume-docs/src/docs/UserGuide/RateLimit
 create mode 100644 flume-docs/src/docs/UserGuide/Security
 create mode 100644 flume-docs/src/docs/UserGuide/Troubleshooting
 create mode 100644 flume-docs/src/docs/UserGuide/UsingOutput
 create mode 100644 flume-docs/src/docs/UserGuide/VersionHistory
 create mode 100644 flume-docs/src/docs/Windows/Windows
 create mode 100755 flume-docs/src/docs/Windows/flume-logo.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup-2-license.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup-3-components.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup-4-master.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup-5-jre.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup-6-installdir.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup-7-complete.png
 create mode 100755 flume-docs/src/docs/Windows/flume-setup.png
 create mode 100755 flume-docs/src/docs/Windows/flume-tray-icon.png
 create mode 100755 flume-docs/src/docs/Windows/flume-tray-menu.png
 create mode 100644 flume-docs/src/docs/cookbook/Apache
 create mode 100644 flume-docs/src/docs/cookbook/Cookbook
 create mode 100644 flume-docs/src/docs/cookbook/Scribe
 create mode 100644 flume-docs/src/docs/cookbook/Syslog
 create mode 100644 flume-docs/src/docs/cookbook/Testing
 create mode 100644 flume-docs/src/docs/web/breadcrumbs.xsl
 create mode 100644 flume-docs/src/docs/web/docbook.css
 create mode 100644 flume-docs/src/docs/web/footer.xsl
 create mode 100644 flume-docs/src/docs/web/header.xsl
 create mode 100644 flume-docs/src/docs/web/html.xsl
 create mode 100644 flume-docs/src/docs/web/images/README
 create mode 100644 flume-docs/src/docs/web/images/callouts/1.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/10.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/11.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/12.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/13.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/14.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/15.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/2.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/3.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/4.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/5.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/6.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/7.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/8.png
 create mode 100644 flume-docs/src/docs/web/images/callouts/9.png
 create mode 100644 flume-docs/src/docs/web/images/caution.png
 create mode 100644 flume-docs/src/docs/web/images/example.png
 create mode 100644 flume-docs/src/docs/web/images/home.png
 create mode 100644 flume-docs/src/docs/web/images/important.png
 create mode 100644 flume-docs/src/docs/web/images/next.png
 create mode 100644 flume-docs/src/docs/web/images/note.png
 create mode 100644 flume-docs/src/docs/web/images/prev.png
 create mode 100644 flume-docs/src/docs/web/images/tip.png
 create mode 100644 flume-docs/src/docs/web/images/up.png
 create mode 100644 flume-docs/src/docs/web/images/warning.png
 create mode 100644 flume-docs/src/docs/web/index.txt

diff --git a/docs/Cookbook/breadcrumbs.xsl b/docs/Cookbook/breadcrumbs.xsl
deleted file mode 100644
index 98d70ba..0000000
--- a/docs/Cookbook/breadcrumbs.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<xsl:template name="breadcrumbs">
-  <xsl:param name="this.node" select="."/>
-  <div class="breadcrumbs">
-    <xsl:for-each select="$this.node/ancestor::*">
-      <span class="breadcrumb-link">
-        <a>
-          <xsl:attribute name="href">
-            <xsl:call-template name="href.target">
-              <xsl:with-param name="object" select="."/>
-              <xsl:with-param name="context" select="$this.node"/>
-            </xsl:call-template>
-          </xsl:attribute>
-          <xsl:apply-templates select="." mode="title.markup"/>
-        </a>
-      </span>
-      <xsl:text> &gt; </xsl:text>
-    </xsl:for-each>
-    <!-- And display the current node, but not as a link -->
-    <span class="breadcrumb-node">
-      <xsl:apply-templates select="$this.node" mode="title.markup"/>
-    </span>
-  </div>
-</xsl:template>
-
-</xsl:stylesheet>
diff --git a/docs/Cookbook/docbook.css b/docs/Cookbook/docbook.css
deleted file mode 100644
index a622c9f..0000000
--- a/docs/Cookbook/docbook.css
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- *
- *
- * This file was provided to this project under these terms:
- *
- * Copyright (c) 2001, 2003 The FreeBSD Documentation Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD: doc/share/misc/docbook.css,v 1.14 2008/11/21 07:28:34 keramida Exp $
- */
-
-BODY ADDRESS {
-	line-height: 1.3;
-	margin: .6em 0;
-}
-
-BODY BLOCKQUOTE {
-	margin-top: .75em;
-	line-height: 1.5;
-	margin-bottom: .75em;
-}
-
-HTML BODY {
-	margin: 1em 4% 1em 4%;
-	line-height: 1.2;
-	font-family: verdana;
-}
-
-a {
-    text-decoration: none;
-    color: #1c84bf;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-a:visited {
-    color: #597e93;
-}
-
-div.titlepage .title {
-	font-size: 26px;
-}
-
-.LEGALNOTICE {
-	font-size: small;
-	font-variant: small-caps;
-}
-
-BODY DIV {
-	margin: 0;
-}
-
-DL {
-	margin: .8em 0;
-	line-height: 1.2;
-}
-
-BODY FORM {
-	margin: .6em 0;
-}
-
-H1, H2, H3, H4, H5, H6,
-DIV.EXAMPLE P B,
-.QUESTION,
-DIV.TABLE P B,
-DIV.PROCEDURE P B {
-	color: #148BCF;
-}
-
-BODY H1, BODY H2, BODY H3, BODY H4, BODY H5, BODY H6 {
-	line-height: 1.3;
-	margin-left: 0;
-}
-
-BODY H1, H2, H3, H4, H5, H6 {
-	margin: 6px 0 0 0;
-}
-
-BODY HR {
-	border-width: 0 0 1px 0;
-	border-style: solid;
-	border-color: #0068BB;
-}
-
-BODY IMG.NAVHEADER {
-	margin: 0 0 0 -4%;
-}
-
-OL {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-BODY PRE {
-	margin: .75em 0;
-	line-height: 1.0;
-	font-family: monospace;
-}
-
-BODY TD, BODY TH {
-	line-height: 1.2;
-}
-
-UL, BODY DIR, BODY MENU {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-HTML {
-	margin: 0; 
-	padding: 0;
-}
-
-BODY P B.APPLICATION {
-	color: #000000;
-}
-
-.navfooter td, .literal, .FILENAME {
-	color: #007a00;
-	font-family: monospace;
-}
-
-
-.GUIMENU, .GUIMENUITEM, .GUISUBMENU,
-.GUILABEL, .INTERFACE,
-.SHORTCUT, .SHORTCUT .KEYCAP {
-	font-weight: bold;
-}
-
-.GUIBUTTON {
-	background-color: #CFCFCF;
-	padding: 2px;
-}
-
-.ACCEL {
-	background-color: #F0F0F0;
-	text-decoration: underline;
-} 
-
-.PROGRAMLISTING {
-	padding: 1ex;
-	background-color: #eee;
-	border: 1px solid #ccc;
-}
-
-@media screen {  /* hide from IE3 */:/
-	a[href]:hover { background: #ffa }
-}
-
-.NOTE, .SCREEN {
-	color: #222;
-	background: #fff;
-	border: 1px solid #ccc;
-	padding: 0.4em 0.4em;
-}
-
-.SCREEN {
-        background: #eee;
-}
-
-.TIP {
-	color: #004F00;
-	background: #fff;
-	border: 1px solid green;
-	padding: 0.4em 0.4em;
-}
-
-table.CDH {
-	padding:0px 0px 8px
-}
-
-table.CDH th {
-	background: #EDF2F9;
-	border-right: 0.5pt solid #adb6cd;
-	border-bottom: 0.5pt solid #adb6cd;
-        padding:2px 8px 1px;
-}
-
-table.CDH td {
-	font-size: 12px;
-}
-
-div.breadcrumbs {
-	border-top: 2px solid #0068BB;
-        background: #EDF2F9;
-	font-style: bold;
-	font-size: small;
-	padding: 0.6em 0.6em;
-}
-
-span.breadcrumb-node {
-	font-style: italic;
-}
-
-/* Match the the breadcrumbs at the top */
-div.footer-text {
-	border-bottom: 2px solid #0068BB;
-        background: #EDF2F9;
-        font-size: small;
-        padding: 0.2em 0.2em;
-	text-align: center;
-}
-
-.IMPORTANT, .caution {
-	font-style:italic;
-	border: 1px solid #a00;
-	border-left: 12px solid #c00;
-	padding: 0.1em 1em;
-}
-
-.WARNING {
-	color: #9F1313;
-	background: #fff;
-	border: 1px solid #e59595;
-	padding: 0.4em 0.4em;
-	width: 85%;
-}
-
-.EXAMPLE {
-	width: 100%;
-}
-
-.tip, .note, .caution {
-	margin:4px 0 4px;
-}
-
-img {
-	border: 0;
-}
diff --git a/docs/Cookbook/footer.xsl b/docs/Cookbook/footer.xsl
deleted file mode 100644
index 98582c0..0000000
--- a/docs/Cookbook/footer.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:template name="user.footer.content">
-
-  <script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script>
-  <script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script>
-
-  <div class="footer-text">
-  <span align="center"><a href="index.html"><img src="images/home.png"
-      alt="Documentation Home" /></a></span>
-  <br/>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/Cookbook/header.xsl b/docs/Cookbook/header.xsl
deleted file mode 100644
index 7c74909..0000000
--- a/docs/Cookbook/header.xsl
+++ /dev/null
@@ -1,41 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:import href="breadcrumbs.xsl"/>
-
-<xsl:template name="user.head.content">
-</xsl:template>
-
-<xsl:template name="user.header.content">
-
-    <div style="clear:both; margin-bottom: 4px" />
-    <div align="center">
-      <a href="index.html"><img src="images/home.png"
-          alt="Documentation Home" /></a>
-    </div>
-    <span class="breadcrumbs">
-    <xsl:call-template name="breadcrumbs"/>
-    </span>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/Cookbook/html.xsl b/docs/Cookbook/html.xsl
deleted file mode 100644
index 2617dd4..0000000
--- a/docs/Cookbook/html.xsl
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version='1.0'?>
-
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- $FreeBSD: doc/share/xsl/freebsd-html.xsl,v 1.1 2003/01/03 05:06:14 trhodes Exp $ -->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                version='1.0'
-                xmlns="http://www.w3.org/TR/xhtml1/transitional"
-                exclude-result-prefixes="#default">
-
-  <xsl:import href="header.xsl"/>
-  <xsl:import href="footer.xsl"/>
-
-  <xsl:template name="body.attributes" />
-
-  <xsl:param name="html.stylesheet" select="'docbook.css'"/>
-  <xsl:param name="use.id.as.filename" select="1"/>
-  <xsl:param name="generate.legalnotice.link" select="'1'"/>
-  <xsl:param name="generate.section.toc.level" select="100"></xsl:param>
-
-  <xsl:param name="toc.section.depth" select="10"/>
-  <xsl:param name="section.autolabel" select="1"/>
-  <xsl:param name="section.label.includes.component.label" select="1"/>
-  <xsl:param name="chunk.section.depth" select="100"></xsl:param>
-  <xsl:param name="chunk.first.sections" select="1"></xsl:param>
-  <xsl:param name="navig.showtitles" select="1"></xsl:param>
-
-  <xsl:param name="admon.graphics" select="1"></xsl:param>
-  <xsl:param name="admon.graphics.extension">.png</xsl:param>
-  <xsl:param name="admon.graphics.path">images/</xsl:param>
-
-  <xsl:param name="navig.graphics" select="1"></xsl:param>
-  <xsl:param name="navig.graphics.extension">.png</xsl:param>
-  <xsl:param name="navig.graphics.path">images/</xsl:param>
-
-  <xsl:param name="header.rule" select="0"></xsl:param>
-  <xsl:param name="footer.rule" select="0"></xsl:param>
-  <xsl:param name="suppress.header.navigation" select="1"></xsl:param>
-
-  <xsl:param name="generate.index" select="1"></xsl:param>
-
-  <xsl:param name="spacing.paras" select="0"></xsl:param>
-  <xsl:param name="html.cleanup" select="1"></xsl:param>
-
-  <xsl:param name="table.borders.with.css" select="1"></xsl:param>
-  <!-- xsl:param name="id.warnings" select="1"></xsl:param -->
-
-  <xsl:param name="generate.toc">
-appendix  toc,title
-article   toc,title,figure,equation
-book      toc,title,figure,example,equation
-chapter   toc,title
-part      toc,title
-preface   toc,title
-qandadiv  toc
-qandaset  toc
-reference toc,title
-sect1     toc
-sect2     toc
-sect3     toc
-sect4     toc
-sect5     toc
-section   toc
-set       toc,title
-   </xsl:param>
-
-</xsl:stylesheet>
diff --git a/docs/Cookbook/images/README b/docs/Cookbook/images/README
deleted file mode 100644
index f12b2a7..0000000
--- a/docs/Cookbook/images/README
+++ /dev/null
@@ -1,5 +0,0 @@
-Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook
-icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency
-from the Jimmac icons to get round MS IE and FOP PNG incompatibilies.
-
-Stuart Rackham
diff --git a/docs/Cookbook/images/callouts/1.png b/docs/Cookbook/images/callouts/1.png
deleted file mode 100644
index 7d473430b7bec514f7de12f5769fe7c5859e8c5d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/10.png b/docs/Cookbook/images/callouts/10.png
deleted file mode 100644
index 997bbc8246a316e040e0804174ba260e219d7d33..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/11.png b/docs/Cookbook/images/callouts/11.png
deleted file mode 100644
index ce47dac3f52ac49017749a3fea53db57d006993c..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/12.png b/docs/Cookbook/images/callouts/12.png
deleted file mode 100644
index 31daf4e2f25b6712499ee32de9c2e3b050b691ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/13.png b/docs/Cookbook/images/callouts/13.png
deleted file mode 100644
index 14021a89c2ed3d4881afea6e3a315bce4f95efce..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/14.png b/docs/Cookbook/images/callouts/14.png
deleted file mode 100644
index 64014b75fe2e84d45ed861974c72462727979360..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/15.png b/docs/Cookbook/images/callouts/15.png
deleted file mode 100644
index 0d65765fcf13dcfd87914744dec8bda115e4adf1..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/2.png b/docs/Cookbook/images/callouts/2.png
deleted file mode 100644
index 5d09341b2f6d2ea2d1d5dad5d980f14b4b05dfd2..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/3.png b/docs/Cookbook/images/callouts/3.png
deleted file mode 100644
index ef7b70047158970cf4e09f1bab2954d39c2d596b..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/4.png b/docs/Cookbook/images/callouts/4.png
deleted file mode 100644
index adb8364eb5d21ecdd4086e16110b62ddcb42aa4a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/5.png b/docs/Cookbook/images/callouts/5.png
deleted file mode 100644
index 4d7eb460021e845981861d77614539314f553993..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/6.png b/docs/Cookbook/images/callouts/6.png
deleted file mode 100644
index 0ba694af6c07d947d219b45a629bd32c60a0f5fe..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/7.png b/docs/Cookbook/images/callouts/7.png
deleted file mode 100644
index 472e96f8ac36862c5645732f2fff19d06ee11a8e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/8.png b/docs/Cookbook/images/callouts/8.png
deleted file mode 100644
index 5e60973c213b37df93666c5a00724f34493974ae..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/callouts/9.png b/docs/Cookbook/images/callouts/9.png
deleted file mode 100644
index a0676d26cc2ff1de12c4ecdeefb44a0d71bc6bde..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/caution.png b/docs/Cookbook/images/caution.png
deleted file mode 100644
index cb9d5ea0df3fb7abe744d6ef885847c13b786d4d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/example.png b/docs/Cookbook/images/example.png
deleted file mode 100644
index bba1c0010ddbb148a93e8a442047ca6a68b5ef1f..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/home.png b/docs/Cookbook/images/home.png
deleted file mode 100644
index 37a5231bacc905886c9351a291e0d5eed68dd60e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/important.png b/docs/Cookbook/images/important.png
deleted file mode 100644
index 1096c232959e8cb5c2977b5eb44c1b67173cb422..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/next.png b/docs/Cookbook/images/next.png
deleted file mode 100644
index 64e126bdda3f2e611199744494367ad0c1a1f9ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/note.png b/docs/Cookbook/images/note.png
deleted file mode 100644
index 841820f7c4fb274fc45fae0a0a526a68eb31d85e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/prev.png b/docs/Cookbook/images/prev.png
deleted file mode 100644
index 3e8f12fe24c6e042bc7ef4badb15449f71529ace..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/tip.png b/docs/Cookbook/images/tip.png
deleted file mode 100644
index a3a029d89891bff666fbdc1a082ae4a7951b5abf..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/up.png b/docs/Cookbook/images/up.png
deleted file mode 100644
index 2db1ce62fa575fd990108b15b0de18a64cae19ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/images/warning.png b/docs/Cookbook/images/warning.png
deleted file mode 100644
index 0b0c419df29b7e71ef14ecf5970b2abcd6d5ea01..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/Cookbook/index.html b/docs/Cookbook/index.html
deleted file mode 100644
index dac2c94..0000000
--- a/docs/Cookbook/index.html
+++ /dev/null
@@ -1,289 +0,0 @@
-<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>Flume Cookbook</title><link rel="stylesheet" href="docbook.css" type="text/css"><meta name="generator" content="DocBook XSL Stylesheets V1.75.2"></head><body><div style="clear:both; margin-bottom: 4px"></div><div align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></div><span class="breadcrumbs"><div class="breadcrumbs"><span class="breadcrumb-node">Flume Cookbook</span></div></span><div lang="en" class="article" title="Flume Cookbook"><div class="titlepage"><div><div><h2 class="title"><a name="id467712"></a>Flume Cookbook</h2></div><div><div class="author"><h3 class="author"><span class="firstname">flume-dev@cloudera.org</span></h3></div></div><div><div class="revhistory"><table border="1" width="100%" summary="Revision history"><tr><th align="left" valign="top" colspan="3"><b>Revision History</b></th></tr><tr><td align="left">Revision 0.9.3-SNAPSHOT</td><td align="left">November 16 2010</td><td align="left">F</td></tr></table></div></div></div><hr></div><div class="toc"><p><b>Table of Contents</b></p><dl><dt><span class="section"><a href="#_introduction">1. Introduction</a></span></dt><dt><span class="section"><a href="#_trying_out_flume_sources_and_sinks">2. Trying out Flume sources and sinks</a></span></dt><dd><dl><dt><span class="section"><a href="#_testing_sources">2.1. Testing sources</a></span></dt><dt><span class="section"><a href="#_testing_sinks_and_decorators">2.2. Testing sinks and decorators</a></span></dt><dt><span class="section"><a href="#_monitoring_nodes">2.3. Monitoring nodes</a></span></dt></dl></dd><dt><span class="section"><a href="#_using_flume_agents_for_apache_2_x_web_server_logging">3. Using Flume Agents for Apache 2.x Web Server Logging</a></span></dt><dd><dl><dt><span class="section"><a href="#_if_you_are_using_centos_red_hat_apache_servers">3.1. If you are using CentOS / Red Hat Apache servers</a></span></dt><dt><span class="section"><a href="#_if_you_are_using_ubuntu_servers_apache_servers">3.2. If you are using Ubuntu servers Apache servers</a></span></dt><dt><span class="section"><a href="#_getting_log_entries_from_piped_log_files">3.3. Getting log entries from Piped Log files</a></span></dt><dt><span class="section"><a href="#_using_piped_logs">3.4. Using piped logs</a></span></dt></dl></dd><dt><span class="section"><a href="#_flume_agents_for_syslog_data">4. Flume Agents for Syslog data</a></span></dt><dd><dl><dt><span class="section"><a href="#_tailing_files">4.1. Tailing files</a></span></dt><dt><span class="section"><a href="#_delivering_syslog_events_via_sockets">4.2. Delivering Syslog events via sockets</a></span></dt><dd><dl><dt><span class="section"><a href="#_configuring_literal_syslogd_literal">4.2.1. Configuring <code class="literal">syslogd</code></a></span></dt><dt><span class="section"><a href="#_configuring_literal_rsyslog_literal">4.2.2. Configuring <code class="literal">rsyslog</code></a></span></dt><dt><span class="section"><a href="#_configuring_literal_syslog_ng_literal">4.2.3. Configuring <code class="literal">syslog-ng</code></a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="#_logging_scribe_events_to_a_flume_agent">5. Logging Scribe Events to a Flume Agent</a></span></dt></dl></div><div class="section" title="1.�Introduction"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_introduction"></a>1.�Introduction</h2></div></div></div><p>Flume is a distributed, reliable, and available service for
-efficiently collecting, aggregating, and moving large amounts of log
-data.  Being highly configurable and very extensible means that there
-are many options and thus many decisions that need to be made by an
-operator.  This document is a "cookbook" with "recipes" for getting
-Flume up and running quickly without being overwhelmed with all the
-details.</p><p>This document starts with some basic tips on experimenting with Flume
-nodes, and then three stories about setting up agents for different
-kinds of sources.  Finally we&#8217;ll do a quick setup for collectors and
-how to troubleshoot distributed deployments.</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Basic debugging with one-shot Flume nodes
-</li><li class="listitem">
-Flume agents for Apache web server logging
-</li><li class="listitem">
-Flume agents for syslog / syslog-ng logging
-</li><li class="listitem">
-Flume agents for scribe logging
-
-
-</li></ul></div></div><div class="section" title="2.�Trying out Flume sources and sinks"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_trying_out_flume_sources_and_sinks"></a>2.�Trying out Flume sources and sinks</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_testing_sources">2.1. Testing sources</a></span></dt><dt><span class="section"><a href="#_testing_sinks_and_decorators">2.2. Testing sinks and decorators</a></span></dt><dt><span class="section"><a href="#_monitoring_nodes">2.3. Monitoring nodes</a></span></dt></dl></div><p>First this section will describe some basic tips for testing sources,
-sinks and logical nodes.</p><div class="section" title="2.1.�Testing sources"><div class="titlepage"><div><div><h3 class="title"><a name="_testing_sources"></a>2.1.�Testing sources</h3></div></div></div><p>Testing sources is a straightforward process.  The <code class="literal">flume</code> script has
-a <code class="literal">dump</code> command that allows you test data source configuration by
-displaying data at the console.</p><pre class="screen">flume dump source</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p><code class="literal">source</code> needs to be a single command line argument, so you may
-need to add <span class="emphasis"><em>quotes</em></span> or "quotes" to the argument if it has quotes or
-parenthesis in them.  Using single quotes allow you to use unescaped
-double quotes in the configuration argument. (ex: <code class="literal">'text("/tmp/foo")'</code>
-or <code class="literal">"text(\"/tmp/foo\")"</code>).</p></td></tr></table></div><p>Here are some simple examples to try:</p><pre class="screen">$ flume dump console
-$ flume dump 'text("/path/to/file")'
-$ flume dump 'file("/path/to/file")'
-$ flume dump syslogTcp
-$ flume dump 'syslogUdp(5140)'
-$ flume dump 'tail("/path/to/file")'
-$ flume dump 'tailDir("path/to/dir", "fileregex")'
-$ flume dump 'rpcSource(12346)'</pre><p>Under the covers, this dump command is actually running the <code class="literal">flume
-node_nowatch</code> command with some extra command line parameters.</p><pre class="screen">flume node_nowatch -1 -s -n dump -c "dump: $1 | console;"</pre><p>Here&#8217;s a summary of what the options mean.</p><div class="horizontal"><table style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">-1</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-one shot execution.  This makes the node instance not use the
-heartbeating mechanism to get a config.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">-s</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-starts the Flume node without starting the node&#8217;s http status
-web server.  If the status web server is started, a Flume node&#8217;s
-status server will keep the process alive even if in one-shot mode.
-If the -s flag is specified along with one-shot mode (-1), the Flume
-node will exit after all logical nodes complete.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">-c "node:src|snk;"</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Starts the node with the given configuration
- definition.  NOTE: If not using -1, this will be invalidated
- upon the first heartbeat to the master.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">-n node</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-gives the node the physical name node.
-</p>
-</td></tr></tbody></table></div><p>You can get info on all of the Flume node commands by using this command:</p><pre class="screen">$ flume node -h</pre></div><div class="section" title="2.2.�Testing sinks and decorators"><div class="titlepage"><div><div><h3 class="title"><a name="_testing_sinks_and_decorators"></a>2.2.�Testing sinks and decorators</h3></div></div></div><p>Now that you can test sources, there is only one more step necessary
-to test arbitrary sinks and decorators from the command line.  A sink
-requires data to consume so some common sources used to generate test
-data include synthetic datasets (<code class="literal">asciisynth</code>), the console
-(<code class="literal">console</code>), or files (<code class="literal">text</code>).</p><p>For example, you can use a synthetic source to generate 1000 events,
-each of 100 "random" bytes data to the console. You could use a text
-source to read a file like /etc/services, or you could use the console
-as a source and interactively enter lines of text as events:</p><pre class="screen">$ flume node_nowatch -1 -s -n dump -c 'dump: asciisynth(1000,100) | console;'
-$ flume node_nowatch -1 -s -n dump -c 'dump: text("/etc/services") | console;'
-$ flume node_nowatch -1 -s -n dump -c 'dump: console | console;'</pre><p>You can also use decorators on the sinks you specify.  For example,
-you could rate limit the amount of data that pushed from the source to
-sink by inserting a delay decorator.  In this case, the <span class="emphasis"><em>delay</em></span>
-decorator waits 100ms before sending each synthesized event to the
-console.</p><pre class="screen">$ flume node_nowatch -1 -s -n dump -c 'dump: asciisynth(1000,100) | { delay(100) =&gt; console};'</pre><p>Using the command line, you can send events via the direct best-effort
-(BE) or disk-failover (DFO) agents.  The example below uses the
-<code class="literal">console</code> source so you can interactively generate data to send in BE
-and DFO mode to collectors.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Flume node_nowatch must be used when piping data in to a Flume
-node&#8217;s console source.  The watchdog program does not forward stdin.</p></td></tr></table></div><pre class="screen">$ flume node_nowatch -1 -s -n dump -c 'dump: console | agentBESink("collectorHost");'
-$ flume node_nowatch -1 -s -n dump -c 'dump: console | agentDFOSink("collectorHost");'</pre><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>Since these nodes are executed with configurations entered at
-the command line and never contact the master, they cannot use the
-automatic chains or logical node translations.  Currently, the
-acknowledgements used in E2E mode go through the master piggy-backed
-on node-to-master heartbeats.  Since this mode does not heartbeat, E2E
-mode should not be used.</p></td></tr></table></div><p>Console sources are useful because we can pipe data into Flume
-directly.  The next example pipes data from a program into Flume which
-then delivers it.</p><pre class="screen">$ &lt;external process&gt; | flume node_nowatch -1 -s -n foo -c 'foo:console|agentBESink("collector");'</pre><p>Ideally, you could write data to a named pipe and just have Flume read
-data from a named pipe using <code class="literal">text</code> or <code class="literal">tail</code>.  Unfortunately, this
-version of Flume&#8217;s <code class="literal">text</code> and <code class="literal">tail</code> are not currently compatible with
-named pipes in a Linux environment.  However, you could pipe data to a
-Flume node listening on the stdin console:</p><pre class="screen">$ tail -f namedpipe | flume node_nowatch -1 -s -n foo -c 'foo:console|agentBESink;'</pre><p>Or you can use the exec source to get its output data:</p><pre class="screen">$ flume node_nowatch -1 -s -n bar -c 'bar:exec("cat pipe")|agentBESink;'</pre></div><div class="section" title="2.3.�Monitoring nodes"><div class="titlepage"><div><div><h3 class="title"><a name="_monitoring_nodes"></a>2.3.�Monitoring nodes</h3></div></div></div><p>While outputting data to a console or to a logfile is an effective way
-to verify data transmission, Flume nodes provide a way to monitor the
-state of sources and sinks.  This can be done by looking at the node&#8217;s
-report page.  By default, you can navigate your web browser to the
-nodes TCP 35862 port.  (<a class="ulink" href="http://node:35862/" target="_top">http://node:35862/</a>)</p><p>This page shows counter information about all of the logical nodes on
-the physical node as well as some basic machine metric information.</p><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>If you have multiple physical nodes on a single machine, there
-may be a port conflict.  If you have the auto-find port option on
-(<code class="literal">flume.node.http.autofindport</code>), the physical node will increment the
-port number until it finds a free port it can bind to.</p></td></tr></table></div></div></div><div class="section" title="3.�Using Flume Agents for Apache 2.x Web Server Logging"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_using_flume_agents_for_apache_2_x_web_server_logging"></a>3.�Using Flume Agents for Apache 2.x Web Server Logging</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_if_you_are_using_centos_red_hat_apache_servers">3.1. If you are using CentOS / Red Hat Apache servers</a></span></dt><dt><span class="section"><a href="#_if_you_are_using_ubuntu_servers_apache_servers">3.2. If you are using Ubuntu servers Apache servers</a></span></dt><dt><span class="section"><a href="#_getting_log_entries_from_piped_log_files">3.3. Getting log entries from Piped Log files</a></span></dt><dt><span class="section"><a href="#_using_piped_logs">3.4. Using piped logs</a></span></dt></dl></div><p>To connect Flume to Apache 2.x servers, you will need to</p><p># Configure web log file permissions</p><p># Tail the web logs or use piped logs to enable Flume to get data from
-  the web server.</p><p>This section will step through basic setup on default Ubuntu Lucid and
-default CentOS 5.5 installations.  Then it will describe various ways
-of integrating Flume.</p><div class="section" title="3.1.�If you are using CentOS / Red Hat Apache servers"><div class="titlepage"><div><div><h3 class="title"><a name="_if_you_are_using_centos_red_hat_apache_servers"></a>3.1.�If you are using CentOS / Red Hat Apache servers</h3></div></div></div><p>By default, CentOS&#8217;s Apache writes weblogs to files owned by <code class="literal">root</code>
-and in group <code class="literal">adm</code> in 0644 (rw-r&#8212;r--) mode.  Flume is run as the
-<code class="literal">flume</code> user, so the Flume node is able to read the logs.</p><p>Apache on CentOS/Red Hat servers defaults to writing logs to two
-files:</p><pre class="screen">/var/log/httpd/access_log
-/var/log/httpd/error_log</pre><p>The simplest way to gather data from these files is to tail the files
-by configuring Flume nodes to use Flume&#8217;s <code class="literal">tail</code> source:</p><pre class="screen">tail("/var/log/httpd/access_log")
-tail("/var/log/httpd/error_log")</pre></div><div class="section" title="3.2.�If you are using Ubuntu servers Apache servers"><div class="titlepage"><div><div><h3 class="title"><a name="_if_you_are_using_ubuntu_servers_apache_servers"></a>3.2.�If you are using Ubuntu servers Apache servers</h3></div></div></div><p>By default, Ubuntu writes weblogs to files owned by <code class="literal">root</code> and in
-group <code class="literal">adm</code> in 0640 (rw-r-----) mode.  Flume is run as the <code class="literal">flume</code>
-user and by default will <span class="strong"><strong>not</strong></span> be able to tread the files.  One
-approach to allow the <code class="literal">flume</code> user to read the files is to add it to
-the <code class="literal">adm</code> group.</p><p>Apache servers on Ubuntu defaults to writing logs to three files:</p><pre class="screen">/var/log/apache2/access.log
-/var/log/apache2/error.log
-/var/log/apache2/other_vhosts_access.log</pre><p>The simplest way to gather data from these files is by configuring
-Flume nodes to use Flume&#8217;s <code class="literal">tail</code> source:</p><pre class="screen">tail("/var/log/apache2/access.log")
-tail("/var/log/apache2/error.log")
-tail("/var/log/apache2/other_vhosts_access.log")</pre></div><div class="section" title="3.3.�Getting log entries from Piped Log files"><div class="titlepage"><div><div><h3 class="title"><a name="_getting_log_entries_from_piped_log_files"></a>3.3.�Getting log entries from Piped Log files</h3></div></div></div><p>The Apache 2.x&#8217;s documentation
-(<a class="ulink" href="http://httpd.apache.org/docs/2.2/logs.html" target="_top">http://httpd.apache.org/docs/2.2/logs.html</a>) describes using piped
-logfile with the <code class="literal">CustomLog</code> directive.  Their example uses
-<code class="literal">rotatelogs</code> to periodically write data to new files with a given
-prefix.  Here are some example directives that could be in the
-<code class="literal">httpd.conf</code>/<code class="literal">apache2.conf</code> file.</p><pre class="screen">LogFormat "%h %l %u %t \"%r\" %&gt;s %b" common
-CustomLog "|/usr/sbin/rotatelogs /var/log/apache2/foo_access_log 3600" common</pre><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>In Ubuntu Lucid these directives are in
-<code class="literal">/etc/apache2/sites-available/default</code>. In CentOS 5.5, these
-directives are in <code class="literal">/etc/httpd/conf/httpd.conf</code>.</p></td></tr></table></div><p>These directives configure Apache to write log files in
-<code class="literal">/var/log/apache2/foo_access_log.xxxxx</code> every hour (3600 seconds)
-using the "common" log format.</p><p>You can use Flume&#8217;s <code class="literal">tailDir</code> source to read all files without
-modifing the Apache settings:</p><pre class="screen">tailDir("/var/log/apache2/", "foo_access_log.*")</pre><p>The first argument is the directory, and then the second is a regex
-that should match against the file name.  <code class="literal">tailDir</code> will watch the dir
-and tail all files that have matching file names.</p></div><div class="section" title="3.4.�Using piped logs"><div class="titlepage"><div><div><h3 class="title"><a name="_using_piped_logs"></a>3.4.�Using piped logs</h3></div></div></div><p>Instead of writing data to disk and then having Flume read it, you can
-have Flume ingest data directly from Apache.  To do so, modify the web
-server&#8217;s parameters and use its piped log feature by adding some
-directives to Apache&#8217;s configuration:</p><pre class="screen">CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentBESink(\"collector\");\'" common</pre><pre class="screen">CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentDFOSink(\"collector\");\'" common</pre><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>By default, CentOS does not have the Java required by the
-Flume node in user <code class="literal">root</code>'s path.  You can use <code class="literal">alternatives</code> to
-create a managed symlink in <code class="literal">/usr/bin/</code> for the java executable.</p></td></tr></table></div><p>Using piped logs can be more efficient, but is riskier because Flume
-can deliver messages without saving on disk.  Doing this, however,
-increases the probabiltiy of event loss.  From a security point of
-view, this Flume node instance runs as Apache&#8217;s user which is often
-<code class="literal">root</code> according to the Apache manual.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>You could configure the one-shot mode node to deliver data
-directly to a collector.  This can only be done at the best effort or
-disk-failover level.</p></td></tr></table></div><p>The prior examples use Flume nodes in one-shot mode which runs without
-contacting a master.  Unfortunately, it means that one-shot mode
-cannot directly use the automatic chains or the end-to-end (E2E)
-reliablity mode.  This is because the automatic chains are generated
-by the master and because E2E mode currently delivers acknowledgements
-through the master.</p><p>However, you can have a one-shot Flume node deliver data to a Flume
-local node daemon where the reliable E2E mode can be used.  In this
-setup we would have the following Apache directive:</p><pre class="screen">CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentBESink(\"localhost\", 12345);\'" common</pre><p>Then you can have a Flume node setup to listen with the following
-configuration:</p><pre class="screen">node : rpcSource(12345) | agentE2ESink("collector");</pre><p>Since this daemon node is connected to the master, it can use the
-auto*Chains.</p><pre class="screen">node : rpcSource(12345) | autoE2EChain;</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>End-to-end mode attempts to ensure the deliver of the data that
-enters the E2E sink.  In this one-shot-node to reliable-node scenario,
-data is not safe it gets to the E2E sink.  However, since this is a
-local connection, it should only fail when the machine or processes
-fails.  The one-shot node can be set to disk failover (DFO) mode in
-order to reduce the chance of message loss if the daemon node&#8217;s
-configuration changes.</p></td></tr></table></div></div></div><div class="section" title="4.�Flume Agents for Syslog data"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_flume_agents_for_syslog_data"></a>4.�Flume Agents for Syslog data</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_tailing_files">4.1. Tailing files</a></span></dt><dt><span class="section"><a href="#_delivering_syslog_events_via_sockets">4.2. Delivering Syslog events via sockets</a></span></dt><dd><dl><dt><span class="section"><a href="#_configuring_literal_syslogd_literal">4.2.1. Configuring <code class="literal">syslogd</code></a></span></dt><dt><span class="section"><a href="#_configuring_literal_rsyslog_literal">4.2.2. Configuring <code class="literal">rsyslog</code></a></span></dt><dt><span class="section"><a href="#_configuring_literal_syslog_ng_literal">4.2.3. Configuring <code class="literal">syslog-ng</code></a></span></dt></dl></dd></dl></div><p><code class="literal">syslog</code> is the standard unix single machine logging service.  Events
-are generally emitted as lines with a time stamp, "facility" type,
-priority, and message.  Syslog can be configured to send data to
-remote destinations.  The default syslog remote delivery was
-originally designed to provide best effort delivery service.  Today,
-there are several more advanced syslog services that deliver messages
-with improved reliability (TCP connections with memory buffering on
-failure).  The reliability guarantees however are one hop and weaker
-than Flume&#8217;s more reliable delivery mechanism.</p><p>This section describes collecting syslog data using two methods.  The
-first part describes a file tailing approach.  The latter parts
-describe syslog system configuration guidance that enables directly
-feeding Flume&#8217;s <code class="literal">syslog*</code> sources.</p><div class="section" title="4.1.�Tailing files"><div class="titlepage"><div><div><h3 class="title"><a name="_tailing_files"></a>4.1.�Tailing files</h3></div></div></div><p>The quickest way to record syslog messages is to tail syslog generated
-log files.  These files generally live in <code class="literal">/var/log</code>.</p><p>Some examples include:</p><pre class="screen">/var/log/auth.log
-/var/log/messages
-/var/log/syslog
-/var/log/user.log</pre><p>These files could be tailed by Flume nodes with tail sources:</p><pre class="screen">tail("/var/log/auth.log")
-tail("/var/log/messages")
-tail("/var/log/syslog")
-tail("/var/log/user.log")</pre><p>Depending on your system configuration, there may be permissions
-issues when accessing these files from the Flume node process.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Red Hat/CentOS systems default to writing log files owned by
-root, in group root, and with 0600 (-rw-------) permissions. Flume
-could be run as root, but this is not advised because Flume can be
-remotely configured to execute arbitrary programs.</p></td></tr></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Ubuntu systems default to writing logs files owned by syslog, in
-group adm, and with 0640 (-rw-r-----) permissions.  By adding the user
-"flume" to group "adm", a Flume node running user "flume" should be
-able to read the syslog generated files.</p></td></tr></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>When tailing files, the time when the event is read is used as the
-time stamp.</p></td></tr></table></div></div><div class="section" title="4.2.�Delivering Syslog events via sockets"><div class="titlepage"><div><div><h3 class="title"><a name="_delivering_syslog_events_via_sockets"></a>4.2.�Delivering Syslog events via sockets</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_configuring_literal_syslogd_literal">4.2.1. Configuring <code class="literal">syslogd</code></a></span></dt><dt><span class="section"><a href="#_configuring_literal_rsyslog_literal">4.2.2. Configuring <code class="literal">rsyslog</code></a></span></dt><dt><span class="section"><a href="#_configuring_literal_syslog_ng_literal">4.2.3. Configuring <code class="literal">syslog-ng</code></a></span></dt></dl></div><p>The original syslog listens to the <code class="literal">/dev/log</code> named pipe, and can be
-configured to listen on UDP port
-514. (<a class="ulink" href="http://tools.ietf.org/search/rfc5424" target="_top">http://tools.ietf.org/search/rfc5424</a>). More advanced versions
-(rsyslog, syslog-ng) can send and recieve over TCP and may do
-in-memory queuing/buffering. For example, syslog-ng and rsyslog can
-optionally use the default UDP port 514 or use TCP port 514 for better
-recovery options.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>By default only superusers can listen on on UDP/TCP ports 514.
-Unix systems usually only allow ports &lt;1024 to be bound by
-superusers. While Flume can run as superuser, from a security stance
-this is not advised.  The examples provide directions to route to the
-user-bindable port 5140.</p></td></tr></table></div><p>For debugging syslog configurations, you can just use <span class="emphasis"><em>flume dump</em></span>
-with syslog sources.  This command outputs received syslog data to the
-console.  To test if syslog data is coming in to the proper port you
-can run this command from the command line:</p><pre class="screen">$ flume dump 'syslogUdp(5140)'</pre><p>This will dump all incoming events to the console.</p><p>If you are satisfied with your connection, you can have a Flume node
-run on the machine configure its sink for the reliability level you
-desire.</p><p>Using a <code class="literal">syslog*</code> Flume source will save the entire line of event
-data, use the timestamp found in the original data, extract a <code class="literal">host</code>,
-and attempt to extract a service from the syslog line.  All of these
-map to a Flume event&#8217;s fields except for <code class="literal">service</code> so this is added as
-extra metadata field to each event (this is a convention with
-syslog defined in RFC).</p><p>So, a syslog entry whose body is this:</p><pre class="screen">Sep 14 07:57:24 blitzwing dhclient: bound to 192.168.126.212 -- renewal in 710 seconds.</pre><p>will have the Flume event body:</p><pre class="screen">Sep 14 07:57:24 blitzwing dhclient: bound to 192.168.126.212 -- renewal in 710 seconds.</pre><p>The event will also translated the "Sep 14 07:57:24" date+time data so
-that it will be bucketable.  Since this date does not have a year, it
-assumes the current year and since it has no timezone it assumes the
-local timezone.  The host field should be "blitzwing", and the
-optional "service" metadata field will contain "dhclient".</p><div class="section" title="4.2.1.�Configuring syslogd"><div class="titlepage"><div><div><h4 class="title"><a name="_configuring_literal_syslogd_literal"></a>4.2.1.�Configuring <code class="literal">syslogd</code></h4></div></div></div><p>The original syslog is <code class="literal">syslogd</code>.  It is configured by an
-<code class="literal">/etc/syslog.conf</code> file.  Its format is fairly simple.</p><p>Syslog recieves messages and then sends to out to different facilities
-that have associated names
-(<a class="ulink" href="http://tools.ietf.org/search/rfc5424#section-6.2" target="_top">http://tools.ietf.org/search/rfc5424#section-6.2</a>).</p><p>The <code class="literal">/etc/syslog.conf</code> file essentially contains lists of facilities
-and "actions".  These "actions" are destinations such as regular
-files, but can also be named pipes, consoles, or remote machines.  One
-can specify a remote machine by prefixing an <span class="emphasis"><em>@</em></span> symbol in front the
-destination host machine.  If no port is specified, events are sent
-via UDP port 514.</p><p>The example below specifies delivery to machine localhost on port
-5140.</p><pre class="screen">user.*     @localhost:5140</pre><p>A Flume node daemon running on this machine would have a <code class="literal">syslogUdp</code>
-source listening for new log data.</p><pre class="screen">host-syslog : syslogUdp(5140) | autoE2EChain ;</pre></div><div class="section" title="4.2.2.�Configuring rsyslog"><div class="titlepage"><div><div><h4 class="title"><a name="_configuring_literal_rsyslog_literal"></a>4.2.2.�Configuring <code class="literal">rsyslog</code></h4></div></div></div><p><code class="literal">rsyslog</code> is a more advanced drop-in replacement for syslog and the
-default syslog system used by Ubuntu systems.  It supports basic
-filtering, best effort delivery, and queuing for handling one-hop
-downstream failures.</p><p><code class="literal">rsyslog</code> actually extends the syslog configuration file format.
-Similar to regular <code class="literal">syslogd</code> you can send data to a remote machine on
-listening on UDP port 514 (standard syslog port).</p><pre class="screen">*.*   @remotehost</pre><p>Moreover, <code class="literal">rsyslog</code> also allows you to use the more reliable TCP
-protocol to send data to a remote host listening on TCP port 514.  In
-<code class="literal">rsyslog</code> configurations, an <span class="emphasis"><em>@@</em></span> prefix dictates the use of TCP.</p><pre class="screen">*.*  @@remotehost</pre><p>Similarly, you can also append a suffix port number to have it deliver
-to a particular port.  In this example, events are delivered to
-localhost TCP port 5140.</p><pre class="screen">*.*  @@localhost:5140</pre><p>Assuming you have a Flume node daemon running on the local host, you
-can capture syslog data by adding a logical node with the following
-configuration:</p><pre class="screen">host-syslog : syslogTcp(5140) | autoE2EChain ;</pre></div><div class="section" title="4.2.3.�Configuring syslog-ng"><div class="titlepage"><div><div><h4 class="title"><a name="_configuring_literal_syslog_ng_literal"></a>4.2.3.�Configuring <code class="literal">syslog-ng</code></h4></div></div></div><p>Syslog-ng is another common replacement for the default syslog logging
-system.  Syslog-ng has a different configuration file format but
-essentially gives the operator the ability to send syslog data from
-different facilities to different remote destinations.  TCP or UDP can
-be used.</p><p>Here is an example of modifications to a <code class="literal">syslog-ng.conf</code> (often found
-in <code class="literal">/etc/syslog-ng/</code>) file.</p><pre class="screen">## set up logging to loghost (which is flume)
-destination loghost {
-        tcp("localhost" port(5140));
-};
-
-# send everything to loghost, too
-log {
-        source(src);
-        destination(loghost);
-};</pre><p>Assuming you have a Flume node daemon running on the local host, you
-can capture syslog data by adding a logical node with the following
-configuration:</p><pre class="screen">host-syslog : syslogTcp(5140) | autoE2EChain ;</pre></div></div></div><div class="section" title="5.�Logging Scribe Events to a Flume Agent"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_logging_scribe_events_to_a_flume_agent"></a>5.�Logging Scribe Events to a Flume Agent</h2></div></div></div><p>Flume can emulate a downstream node for applications that log using
-Scribe.  Scribe is Facebook&#8217;s open source log aggregation framework.
-It has a simple API and uses Thrift as its core network transport
-mechanism.  Flume uses the same Thrift IDL file and can listen for
-data provided by Scribe sources.</p><p>Scribe comes with an example application called <code class="literal">scribe_cat</code>.  It acts
-like the unix <code class="literal">cat</code> program but sends data to a downstream host with a
-specified "category".  Scribe by default uses TCP port 1463.</p><p>You can configure a Flume node to listen for incoming Scribe traffic
-by creating a logical node that uses the <code class="literal">scribe</code> source.  We can then
-assign an arbitrary sink to the node.  In the example below, the
-Scribe nodes receives events, send its events to both the console and
-an automatically-assigned end-to-end agent which delivers the events
-downstream to its collector pools.</p><pre class="screen">scribe: scribe | [console, autoE2EChain];</pre><p>Assuming that this node communicates with the master, and that the
-node is on <code class="literal">localhost</code>, we can use the <code class="literal">scribe_cat</code> program to send
-data to Flume.</p><pre class="screen">$ echo "foo" | scribe_cat localhost testcategory</pre><p>When the Scribe source accepts the Scribe events, it converts Scribe&#8217;s
-category information into a new Flume event metadata entry, and then
-delivers the event to its sinks.  Since Scribe does not include time
-metadata, the timestamp of the created Flume event will be the arrival
-time of the Scribe event.</p><pre class="literallayout">     ______
-    / ___//_  ______  ____
-   / /_/ / / / /    \/ __/
-  / __/ / /_/ / / / / __/
- / / /_/\____/_/_/_/\__/
-/_/ Distributed Log Collection.</pre></div></div><script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script><script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script><div class="footer-text"><span align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></span><br>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div></body></html>
diff --git a/docs/Cookbook/index.txt b/docs/Cookbook/index.txt
deleted file mode 100644
index 04ff317..0000000
--- a/docs/Cookbook/index.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-
-////
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-////
-
-Flume Documentation (v{revnumber})
-==================================
-
-The following pages are the documentation for Flume v{revnumber}:
-
-- link:UserGuide/index.html[Flume User Guide]
-- link:DeveloperGuide/index.html[Flume Developer Guide]
-- link:Cookbook/index.html[Flume Cookbook]
-
diff --git a/docs/Cookbook/index.xml b/docs/Cookbook/index.xml
deleted file mode 100644
index 319a947..0000000
--- a/docs/Cookbook/index.xml
+++ /dev/null
@@ -1,493 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-<?asciidoc-toc?>
-<?asciidoc-numbered?>
-
-<article lang="en">
-<articleinfo>
-    <title>Flume Cookbook</title>
-    <date>November 16 2010</date>
-    <author>
-        <firstname>flume-dev@cloudera.org</firstname>
-    </author>
-    <authorinitials>F</authorinitials>
-<revhistory><revision><revnumber>0.9.3-SNAPSHOT</revnumber><date>November 16 2010</date><authorinitials>F</authorinitials></revision></revhistory>
-</articleinfo>
-<section id="_introduction">
-<title>Introduction</title>
-<simpara>Flume is a distributed, reliable, and available service for
-efficiently collecting, aggregating, and moving large amounts of log
-data.  Being highly configurable and very extensible means that there
-are many options and thus many decisions that need to be made by an
-operator.  This document is a "cookbook" with "recipes" for getting
-Flume up and running quickly without being overwhelmed with all the
-details.</simpara>
-<simpara>This document starts with some basic tips on experimenting with Flume
-nodes, and then three stories about setting up agents for different
-kinds of sources.  Finally we&#8217;ll do a quick setup for collectors and
-how to troubleshoot distributed deployments.</simpara>
-<itemizedlist>
-<listitem>
-<simpara>
-Basic debugging with one-shot Flume nodes
-</simpara>
-</listitem>
-<listitem>
-<simpara>
-Flume agents for Apache web server logging
-</simpara>
-</listitem>
-<listitem>
-<simpara>
-Flume agents for syslog / syslog-ng logging
-</simpara>
-</listitem>
-<listitem>
-<simpara>
-Flume agents for scribe logging
-
-
-</simpara>
-</listitem>
-</itemizedlist>
-</section>
-<section id="_trying_out_flume_sources_and_sinks">
-<title>Trying out Flume sources and sinks</title>
-<simpara>First this section will describe some basic tips for testing sources,
-sinks and logical nodes.</simpara>
-<section id="_testing_sources">
-<title>Testing sources</title>
-<simpara>Testing sources is a straightforward process.  The <literal>flume</literal> script has
-a <literal>dump</literal> command that allows you test data source configuration by
-displaying data at the console.</simpara>
-<screen>flume dump source</screen>
-<note><simpara><literal>source</literal> needs to be a single command line argument, so you may
-need to add <emphasis>quotes</emphasis> or "quotes" to the argument if it has quotes or
-parenthesis in them.  Using single quotes allow you to use unescaped
-double quotes in the configuration argument. (ex: <literal>'text("/tmp/foo")'</literal>
-or <literal>"text(\"/tmp/foo\")"</literal>).</simpara></note>
-<simpara>Here are some simple examples to try:</simpara>
-<screen>$ flume dump console
-$ flume dump 'text("/path/to/file")'
-$ flume dump 'file("/path/to/file")'
-$ flume dump syslogTcp
-$ flume dump 'syslogUdp(5140)'
-$ flume dump 'tail("/path/to/file")'
-$ flume dump 'tailDir("path/to/dir", "fileregex")'
-$ flume dump 'rpcSource(12346)'</screen>
-<simpara>Under the covers, this dump command is actually running the <literal>flume
-node_nowatch</literal> command with some extra command line parameters.</simpara>
-<screen>flume node_nowatch -1 -s -n dump -c "dump: $1 | console;"</screen>
-<simpara>Here&#8217;s a summary of what the options mean.</simpara>
-<informaltable tabstyle="horizontal" frame="none" colsep="0" rowsep="0"><tgroup cols="2"><colspec colwidth="15*"/><colspec colwidth="85*"/><tbody valign="top">
-<row>
-<entry>
-<simpara>
-<literal>-1</literal> 
-</simpara>
-</entry>
-<entry>
-<simpara>
-one shot execution.  This makes the node instance not use the
-heartbeating mechanism to get a config.
-</simpara>
-</entry>
-</row>
-<row>
-<entry>
-<simpara>
-<literal>-s</literal> 
-</simpara>
-</entry>
-<entry>
-<simpara>
-starts the Flume node without starting the node&#8217;s http status
-web server.  If the status web server is started, a Flume node&#8217;s
-status server will keep the process alive even if in one-shot mode.
-If the -s flag is specified along with one-shot mode (-1), the Flume
-node will exit after all logical nodes complete.
-</simpara>
-</entry>
-</row>
-<row>
-<entry>
-<simpara>
-<literal>-c "node:src|snk;"</literal> 
-</simpara>
-</entry>
-<entry>
-<simpara>
-Starts the node with the given configuration
- definition.  NOTE: If not using -1, this will be invalidated
- upon the first heartbeat to the master.
-</simpara>
-</entry>
-</row>
-<row>
-<entry>
-<simpara>
-<literal>-n node</literal> 
-</simpara>
-</entry>
-<entry>
-<simpara>
-gives the node the physical name node.
-</simpara>
-</entry>
-</row>
-</tbody></tgroup></informaltable>
-<simpara>You can get info on all of the Flume node commands by using this command:</simpara>
-<screen>$ flume node -h</screen>
-</section>
-<section id="_testing_sinks_and_decorators">
-<title>Testing sinks and decorators</title>
-<simpara>Now that you can test sources, there is only one more step necessary
-to test arbitrary sinks and decorators from the command line.  A sink
-requires data to consume so some common sources used to generate test
-data include synthetic datasets (<literal>asciisynth</literal>), the console
-(<literal>console</literal>), or files (<literal>text</literal>).</simpara>
-<simpara>For example, you can use a synthetic source to generate 1000 events,
-each of 100 "random" bytes data to the console. You could use a text
-source to read a file like /etc/services, or you could use the console
-as a source and interactively enter lines of text as events:</simpara>
-<screen>$ flume node_nowatch -1 -s -n dump -c 'dump: asciisynth(1000,100) | console;'
-$ flume node_nowatch -1 -s -n dump -c 'dump: text("/etc/services") | console;'
-$ flume node_nowatch -1 -s -n dump -c 'dump: console | console;'</screen>
-<simpara>You can also use decorators on the sinks you specify.  For example,
-you could rate limit the amount of data that pushed from the source to
-sink by inserting a delay decorator.  In this case, the <emphasis>delay</emphasis>
-decorator waits 100ms before sending each synthesized event to the
-console.</simpara>
-<screen>$ flume node_nowatch -1 -s -n dump -c 'dump: asciisynth(1000,100) | { delay(100) =&gt; console};'</screen>
-<simpara>Using the command line, you can send events via the direct best-effort
-(BE) or disk-failover (DFO) agents.  The example below uses the
-<literal>console</literal> source so you can interactively generate data to send in BE
-and DFO mode to collectors.</simpara>
-<note><simpara>Flume node_nowatch must be used when piping data in to a Flume
-node&#8217;s console source.  The watchdog program does not forward stdin.</simpara></note>
-<screen>$ flume node_nowatch -1 -s -n dump -c 'dump: console | agentBESink("collectorHost");'
-$ flume node_nowatch -1 -s -n dump -c 'dump: console | agentDFOSink("collectorHost");'</screen>
-<warning><simpara>Since these nodes are executed with configurations entered at
-the command line and never contact the master, they cannot use the
-automatic chains or logical node translations.  Currently, the
-acknowledgements used in E2E mode go through the master piggy-backed
-on node-to-master heartbeats.  Since this mode does not heartbeat, E2E
-mode should not be used.</simpara></warning>
-<simpara>Console sources are useful because we can pipe data into Flume
-directly.  The next example pipes data from a program into Flume which
-then delivers it.</simpara>
-<screen>$ &lt;external process&gt; | flume node_nowatch -1 -s -n foo -c 'foo:console|agentBESink("collector");'</screen>
-<simpara>Ideally, you could write data to a named pipe and just have Flume read
-data from a named pipe using <literal>text</literal> or <literal>tail</literal>.  Unfortunately, this
-version of Flume&#8217;s <literal>text</literal> and <literal>tail</literal> are not currently compatible with
-named pipes in a Linux environment.  However, you could pipe data to a
-Flume node listening on the stdin console:</simpara>
-<screen>$ tail -f namedpipe | flume node_nowatch -1 -s -n foo -c 'foo:console|agentBESink;'</screen>
-<simpara>Or you can use the exec source to get its output data:</simpara>
-<screen>$ flume node_nowatch -1 -s -n bar -c 'bar:exec("cat pipe")|agentBESink;'</screen>
-</section>
-<section id="_monitoring_nodes">
-<title>Monitoring nodes</title>
-<simpara>While outputting data to a console or to a logfile is an effective way
-to verify data transmission, Flume nodes provide a way to monitor the
-state of sources and sinks.  This can be done by looking at the node&#8217;s
-report page.  By default, you can navigate your web browser to the
-nodes TCP 35862 port.  (<ulink url="http://node:35862/">http://node:35862/</ulink>)</simpara>
-<simpara>This page shows counter information about all of the logical nodes on
-the physical node as well as some basic machine metric information.</simpara>
-<tip><simpara>If you have multiple physical nodes on a single machine, there
-may be a port conflict.  If you have the auto-find port option on
-(<literal>flume.node.http.autofindport</literal>), the physical node will increment the
-port number until it finds a free port it can bind to.</simpara></tip>
-</section>
-</section>
-<section id="_using_flume_agents_for_apache_2_x_web_server_logging">
-<title>Using Flume Agents for Apache 2.x Web Server Logging</title>
-<simpara>To connect Flume to Apache 2.x servers, you will need to</simpara>
-<simpara># Configure web log file permissions</simpara>
-<simpara># Tail the web logs or use piped logs to enable Flume to get data from
-  the web server.</simpara>
-<simpara>This section will step through basic setup on default Ubuntu Lucid and
-default CentOS 5.5 installations.  Then it will describe various ways
-of integrating Flume.</simpara>
-<section id="_if_you_are_using_centos_red_hat_apache_servers">
-<title>If you are using CentOS / Red Hat Apache servers</title>
-<simpara>By default, CentOS&#8217;s Apache writes weblogs to files owned by <literal>root</literal>
-and in group <literal>adm</literal> in 0644 (rw-r&#8212;r--) mode.  Flume is run as the
-<literal>flume</literal> user, so the Flume node is able to read the logs.</simpara>
-<simpara>Apache on CentOS/Red Hat servers defaults to writing logs to two
-files:</simpara>
-<screen>/var/log/httpd/access_log
-/var/log/httpd/error_log</screen>
-<simpara>The simplest way to gather data from these files is to tail the files
-by configuring Flume nodes to use Flume&#8217;s <literal>tail</literal> source:</simpara>
-<screen>tail("/var/log/httpd/access_log")
-tail("/var/log/httpd/error_log")</screen>
-</section>
-<section id="_if_you_are_using_ubuntu_servers_apache_servers">
-<title>If you are using Ubuntu servers Apache servers</title>
-<simpara>By default, Ubuntu writes weblogs to files owned by <literal>root</literal> and in
-group <literal>adm</literal> in 0640 (rw-r-----) mode.  Flume is run as the <literal>flume</literal>
-user and by default will <emphasis role="strong">not</emphasis> be able to tread the files.  One
-approach to allow the <literal>flume</literal> user to read the files is to add it to
-the <literal>adm</literal> group.</simpara>
-<simpara>Apache servers on Ubuntu defaults to writing logs to three files:</simpara>
-<screen>/var/log/apache2/access.log
-/var/log/apache2/error.log
-/var/log/apache2/other_vhosts_access.log</screen>
-<simpara>The simplest way to gather data from these files is by configuring
-Flume nodes to use Flume&#8217;s <literal>tail</literal> source:</simpara>
-<screen>tail("/var/log/apache2/access.log")
-tail("/var/log/apache2/error.log")
-tail("/var/log/apache2/other_vhosts_access.log")</screen>
-</section>
-<section id="_getting_log_entries_from_piped_log_files">
-<title>Getting log entries from Piped Log files</title>
-<simpara>The Apache 2.x&#8217;s documentation
-(<ulink url="http://httpd.apache.org/docs/2.2/logs.html">http://httpd.apache.org/docs/2.2/logs.html</ulink>) describes using piped
-logfile with the <literal>CustomLog</literal> directive.  Their example uses
-<literal>rotatelogs</literal> to periodically write data to new files with a given
-prefix.  Here are some example directives that could be in the
-<literal>httpd.conf</literal>/<literal>apache2.conf</literal> file.</simpara>
-<screen>LogFormat "%h %l %u %t \"%r\" %&gt;s %b" common
-CustomLog "|/usr/sbin/rotatelogs /var/log/apache2/foo_access_log 3600" common</screen>
-<tip><simpara>In Ubuntu Lucid these directives are in
-<literal>/etc/apache2/sites-available/default</literal>. In CentOS 5.5, these
-directives are in <literal>/etc/httpd/conf/httpd.conf</literal>.</simpara></tip>
-<simpara>These directives configure Apache to write log files in
-<literal>/var/log/apache2/foo_access_log.xxxxx</literal> every hour (3600 seconds)
-using the "common" log format.</simpara>
-<simpara>You can use Flume&#8217;s <literal>tailDir</literal> source to read all files without
-modifing the Apache settings:</simpara>
-<screen>tailDir("/var/log/apache2/", "foo_access_log.*")</screen>
-<simpara>The first argument is the directory, and then the second is a regex
-that should match against the file name.  <literal>tailDir</literal> will watch the dir
-and tail all files that have matching file names.</simpara>
-</section>
-<section id="_using_piped_logs">
-<title>Using piped logs</title>
-<simpara>Instead of writing data to disk and then having Flume read it, you can
-have Flume ingest data directly from Apache.  To do so, modify the web
-server&#8217;s parameters and use its piped log feature by adding some
-directives to Apache&#8217;s configuration:</simpara>
-<screen>CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentBESink(\"collector\");\'" common</screen>
-<screen>CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentDFOSink(\"collector\");\'" common</screen>
-<warning><simpara>By default, CentOS does not have the Java required by the
-Flume node in user <literal>root</literal>'s path.  You can use <literal>alternatives</literal> to
-create a managed symlink in <literal>/usr/bin/</literal> for the java executable.</simpara></warning>
-<simpara>Using piped logs can be more efficient, but is riskier because Flume
-can deliver messages without saving on disk.  Doing this, however,
-increases the probabiltiy of event loss.  From a security point of
-view, this Flume node instance runs as Apache&#8217;s user which is often
-<literal>root</literal> according to the Apache manual.</simpara>
-<note><simpara>You could configure the one-shot mode node to deliver data
-directly to a collector.  This can only be done at the best effort or
-disk-failover level.</simpara></note>
-<simpara>The prior examples use Flume nodes in one-shot mode which runs without
-contacting a master.  Unfortunately, it means that one-shot mode
-cannot directly use the automatic chains or the end-to-end (E2E)
-reliablity mode.  This is because the automatic chains are generated
-by the master and because E2E mode currently delivers acknowledgements
-through the master.</simpara>
-<simpara>However, you can have a one-shot Flume node deliver data to a Flume
-local node daemon where the reliable E2E mode can be used.  In this
-setup we would have the following Apache directive:</simpara>
-<screen>CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentBESink(\"localhost\", 12345);\'" common</screen>
-<simpara>Then you can have a Flume node setup to listen with the following
-configuration:</simpara>
-<screen>node : rpcSource(12345) | agentE2ESink("collector");</screen>
-<simpara>Since this daemon node is connected to the master, it can use the
-auto*Chains.</simpara>
-<screen>node : rpcSource(12345) | autoE2EChain;</screen>
-<note><simpara>End-to-end mode attempts to ensure the deliver of the data that
-enters the E2E sink.  In this one-shot-node to reliable-node scenario,
-data is not safe it gets to the E2E sink.  However, since this is a
-local connection, it should only fail when the machine or processes
-fails.  The one-shot node can be set to disk failover (DFO) mode in
-order to reduce the chance of message loss if the daemon node&#8217;s
-configuration changes.</simpara></note>
-</section>
-</section>
-<section id="_flume_agents_for_syslog_data">
-<title>Flume Agents for Syslog data</title>
-<simpara><literal>syslog</literal> is the standard unix single machine logging service.  Events
-are generally emitted as lines with a time stamp, "facility" type,
-priority, and message.  Syslog can be configured to send data to
-remote destinations.  The default syslog remote delivery was
-originally designed to provide best effort delivery service.  Today,
-there are several more advanced syslog services that deliver messages
-with improved reliability (TCP connections with memory buffering on
-failure).  The reliability guarantees however are one hop and weaker
-than Flume&#8217;s more reliable delivery mechanism.</simpara>
-<simpara>This section describes collecting syslog data using two methods.  The
-first part describes a file tailing approach.  The latter parts
-describe syslog system configuration guidance that enables directly
-feeding Flume&#8217;s <literal>syslog*</literal> sources.</simpara>
-<section id="_tailing_files">
-<title>Tailing files</title>
-<simpara>The quickest way to record syslog messages is to tail syslog generated
-log files.  These files generally live in <literal>/var/log</literal>.</simpara>
-<simpara>Some examples include:</simpara>
-<screen>/var/log/auth.log
-/var/log/messages
-/var/log/syslog
-/var/log/user.log</screen>
-<simpara>These files could be tailed by Flume nodes with tail sources:</simpara>
-<screen>tail("/var/log/auth.log")
-tail("/var/log/messages")
-tail("/var/log/syslog")
-tail("/var/log/user.log")</screen>
-<simpara>Depending on your system configuration, there may be permissions
-issues when accessing these files from the Flume node process.</simpara>
-<note><simpara>Red Hat/CentOS systems default to writing log files owned by
-root, in group root, and with 0600 (-rw-------) permissions. Flume
-could be run as root, but this is not advised because Flume can be
-remotely configured to execute arbitrary programs.</simpara></note>
-<note><simpara>Ubuntu systems default to writing logs files owned by syslog, in
-group adm, and with 0640 (-rw-r-----) permissions.  By adding the user
-"flume" to group "adm", a Flume node running user "flume" should be
-able to read the syslog generated files.</simpara></note>
-<note><simpara>When tailing files, the time when the event is read is used as the
-time stamp.</simpara></note>
-</section>
-<section id="_delivering_syslog_events_via_sockets">
-<title>Delivering Syslog events via sockets</title>
-<simpara>The original syslog listens to the <literal>/dev/log</literal> named pipe, and can be
-configured to listen on UDP port
-514. (<ulink url="http://tools.ietf.org/search/rfc5424">http://tools.ietf.org/search/rfc5424</ulink>). More advanced versions
-(rsyslog, syslog-ng) can send and recieve over TCP and may do
-in-memory queuing/buffering. For example, syslog-ng and rsyslog can
-optionally use the default UDP port 514 or use TCP port 514 for better
-recovery options.</simpara>
-<note><simpara>By default only superusers can listen on on UDP/TCP ports 514.
-Unix systems usually only allow ports &lt;1024 to be bound by
-superusers. While Flume can run as superuser, from a security stance
-this is not advised.  The examples provide directions to route to the
-user-bindable port 5140.</simpara></note>
-<simpara>For debugging syslog configurations, you can just use <emphasis>flume dump</emphasis>
-with syslog sources.  This command outputs received syslog data to the
-console.  To test if syslog data is coming in to the proper port you
-can run this command from the command line:</simpara>
-<screen>$ flume dump 'syslogUdp(5140)'</screen>
-<simpara>This will dump all incoming events to the console.</simpara>
-<simpara>If you are satisfied with your connection, you can have a Flume node
-run on the machine configure its sink for the reliability level you
-desire.</simpara>
-<simpara>Using a <literal>syslog*</literal> Flume source will save the entire line of event
-data, use the timestamp found in the original data, extract a <literal>host</literal>,
-and attempt to extract a service from the syslog line.  All of these
-map to a Flume event&#8217;s fields except for <literal>service</literal> so this is added as
-extra metadata field to each event (this is a convention with
-syslog defined in RFC).</simpara>
-<simpara>So, a syslog entry whose body is this:</simpara>
-<screen>Sep 14 07:57:24 blitzwing dhclient: bound to 192.168.126.212 -- renewal in 710 seconds.</screen>
-<simpara>will have the Flume event body:</simpara>
-<screen>Sep 14 07:57:24 blitzwing dhclient: bound to 192.168.126.212 -- renewal in 710 seconds.</screen>
-<simpara>The event will also translated the "Sep 14 07:57:24" date+time data so
-that it will be bucketable.  Since this date does not have a year, it
-assumes the current year and since it has no timezone it assumes the
-local timezone.  The host field should be "blitzwing", and the
-optional "service" metadata field will contain "dhclient".</simpara>
-<section id="_configuring_literal_syslogd_literal">
-<title>Configuring <literal>syslogd</literal></title>
-<simpara>The original syslog is <literal>syslogd</literal>.  It is configured by an
-<literal>/etc/syslog.conf</literal> file.  Its format is fairly simple.</simpara>
-<simpara>Syslog recieves messages and then sends to out to different facilities
-that have associated names
-(<ulink url="http://tools.ietf.org/search/rfc5424#section-6.2">http://tools.ietf.org/search/rfc5424#section-6.2</ulink>).</simpara>
-<simpara>The <literal>/etc/syslog.conf</literal> file essentially contains lists of facilities
-and "actions".  These "actions" are destinations such as regular
-files, but can also be named pipes, consoles, or remote machines.  One
-can specify a remote machine by prefixing an <emphasis>@</emphasis> symbol in front the
-destination host machine.  If no port is specified, events are sent
-via UDP port 514.</simpara>
-<simpara>The example below specifies delivery to machine localhost on port
-5140.</simpara>
-<screen>user.*     @localhost:5140</screen>
-<simpara>A Flume node daemon running on this machine would have a <literal>syslogUdp</literal>
-source listening for new log data.</simpara>
-<screen>host-syslog : syslogUdp(5140) | autoE2EChain ;</screen>
-</section>
-<section id="_configuring_literal_rsyslog_literal">
-<title>Configuring <literal>rsyslog</literal></title>
-<simpara><literal>rsyslog</literal> is a more advanced drop-in replacement for syslog and the
-default syslog system used by Ubuntu systems.  It supports basic
-filtering, best effort delivery, and queuing for handling one-hop
-downstream failures.</simpara>
-<simpara><literal>rsyslog</literal> actually extends the syslog configuration file format.
-Similar to regular <literal>syslogd</literal> you can send data to a remote machine on
-listening on UDP port 514 (standard syslog port).</simpara>
-<screen>*.*   @remotehost</screen>
-<simpara>Moreover, <literal>rsyslog</literal> also allows you to use the more reliable TCP
-protocol to send data to a remote host listening on TCP port 514.  In
-<literal>rsyslog</literal> configurations, an <emphasis>@@</emphasis> prefix dictates the use of TCP.</simpara>
-<screen>*.*  @@remotehost</screen>
-<simpara>Similarly, you can also append a suffix port number to have it deliver
-to a particular port.  In this example, events are delivered to
-localhost TCP port 5140.</simpara>
-<screen>*.*  @@localhost:5140</screen>
-<simpara>Assuming you have a Flume node daemon running on the local host, you
-can capture syslog data by adding a logical node with the following
-configuration:</simpara>
-<screen>host-syslog : syslogTcp(5140) | autoE2EChain ;</screen>
-</section>
-<section id="_configuring_literal_syslog_ng_literal">
-<title>Configuring <literal>syslog-ng</literal></title>
-<simpara>Syslog-ng is another common replacement for the default syslog logging
-system.  Syslog-ng has a different configuration file format but
-essentially gives the operator the ability to send syslog data from
-different facilities to different remote destinations.  TCP or UDP can
-be used.</simpara>
-<simpara>Here is an example of modifications to a <literal>syslog-ng.conf</literal> (often found
-in <literal>/etc/syslog-ng/</literal>) file.</simpara>
-<screen>## set up logging to loghost (which is flume)
-destination loghost {
-        tcp("localhost" port(5140));
-};
-
-# send everything to loghost, too
-log {
-        source(src);
-        destination(loghost);
-};</screen>
-<simpara>Assuming you have a Flume node daemon running on the local host, you
-can capture syslog data by adding a logical node with the following
-configuration:</simpara>
-<screen>host-syslog : syslogTcp(5140) | autoE2EChain ;</screen>
-</section>
-</section>
-</section>
-<section id="_logging_scribe_events_to_a_flume_agent">
-<title>Logging Scribe Events to a Flume Agent</title>
-<simpara>Flume can emulate a downstream node for applications that log using
-Scribe.  Scribe is Facebook&#8217;s open source log aggregation framework.
-It has a simple API and uses Thrift as its core network transport
-mechanism.  Flume uses the same Thrift IDL file and can listen for
-data provided by Scribe sources.</simpara>
-<simpara>Scribe comes with an example application called <literal>scribe_cat</literal>.  It acts
-like the unix <literal>cat</literal> program but sends data to a downstream host with a
-specified "category".  Scribe by default uses TCP port 1463.</simpara>
-<simpara>You can configure a Flume node to listen for incoming Scribe traffic
-by creating a logical node that uses the <literal>scribe</literal> source.  We can then
-assign an arbitrary sink to the node.  In the example below, the
-Scribe nodes receives events, send its events to both the console and
-an automatically-assigned end-to-end agent which delivers the events
-downstream to its collector pools.</simpara>
-<screen>scribe: scribe | [console, autoE2EChain];</screen>
-<simpara>Assuming that this node communicates with the master, and that the
-node is on <literal>localhost</literal>, we can use the <literal>scribe_cat</literal> program to send
-data to Flume.</simpara>
-<screen>$ echo "foo" | scribe_cat localhost testcategory</screen>
-<simpara>When the Scribe source accepts the Scribe events, it converts Scribe&#8217;s
-category information into a new Flume event metadata entry, and then
-delivers the event to its sinks.  Since Scribe does not include time
-metadata, the timestamp of the created Flume event will be the arrival
-time of the Scribe event.</simpara>
-<literallayout class="monospaced">     ______
-    / ___//_  ______  ____
-   / /_/ / / / /    \/ __/
-  / __/ / /_/ / / / / __/
- / / /_/\____/_/_/_/\__/
-/_/ Distributed Log Collection.</literallayout>
-</section>
-</article>
diff --git a/docs/DeveloperGuide/breadcrumbs.xsl b/docs/DeveloperGuide/breadcrumbs.xsl
deleted file mode 100644
index 98d70ba..0000000
--- a/docs/DeveloperGuide/breadcrumbs.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<xsl:template name="breadcrumbs">
-  <xsl:param name="this.node" select="."/>
-  <div class="breadcrumbs">
-    <xsl:for-each select="$this.node/ancestor::*">
-      <span class="breadcrumb-link">
-        <a>
-          <xsl:attribute name="href">
-            <xsl:call-template name="href.target">
-              <xsl:with-param name="object" select="."/>
-              <xsl:with-param name="context" select="$this.node"/>
-            </xsl:call-template>
-          </xsl:attribute>
-          <xsl:apply-templates select="." mode="title.markup"/>
-        </a>
-      </span>
-      <xsl:text> &gt; </xsl:text>
-    </xsl:for-each>
-    <!-- And display the current node, but not as a link -->
-    <span class="breadcrumb-node">
-      <xsl:apply-templates select="$this.node" mode="title.markup"/>
-    </span>
-  </div>
-</xsl:template>
-
-</xsl:stylesheet>
diff --git a/docs/DeveloperGuide/docbook.css b/docs/DeveloperGuide/docbook.css
deleted file mode 100644
index a622c9f..0000000
--- a/docs/DeveloperGuide/docbook.css
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- *
- *
- * This file was provided to this project under these terms:
- *
- * Copyright (c) 2001, 2003 The FreeBSD Documentation Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD: doc/share/misc/docbook.css,v 1.14 2008/11/21 07:28:34 keramida Exp $
- */
-
-BODY ADDRESS {
-	line-height: 1.3;
-	margin: .6em 0;
-}
-
-BODY BLOCKQUOTE {
-	margin-top: .75em;
-	line-height: 1.5;
-	margin-bottom: .75em;
-}
-
-HTML BODY {
-	margin: 1em 4% 1em 4%;
-	line-height: 1.2;
-	font-family: verdana;
-}
-
-a {
-    text-decoration: none;
-    color: #1c84bf;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-a:visited {
-    color: #597e93;
-}
-
-div.titlepage .title {
-	font-size: 26px;
-}
-
-.LEGALNOTICE {
-	font-size: small;
-	font-variant: small-caps;
-}
-
-BODY DIV {
-	margin: 0;
-}
-
-DL {
-	margin: .8em 0;
-	line-height: 1.2;
-}
-
-BODY FORM {
-	margin: .6em 0;
-}
-
-H1, H2, H3, H4, H5, H6,
-DIV.EXAMPLE P B,
-.QUESTION,
-DIV.TABLE P B,
-DIV.PROCEDURE P B {
-	color: #148BCF;
-}
-
-BODY H1, BODY H2, BODY H3, BODY H4, BODY H5, BODY H6 {
-	line-height: 1.3;
-	margin-left: 0;
-}
-
-BODY H1, H2, H3, H4, H5, H6 {
-	margin: 6px 0 0 0;
-}
-
-BODY HR {
-	border-width: 0 0 1px 0;
-	border-style: solid;
-	border-color: #0068BB;
-}
-
-BODY IMG.NAVHEADER {
-	margin: 0 0 0 -4%;
-}
-
-OL {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-BODY PRE {
-	margin: .75em 0;
-	line-height: 1.0;
-	font-family: monospace;
-}
-
-BODY TD, BODY TH {
-	line-height: 1.2;
-}
-
-UL, BODY DIR, BODY MENU {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-HTML {
-	margin: 0; 
-	padding: 0;
-}
-
-BODY P B.APPLICATION {
-	color: #000000;
-}
-
-.navfooter td, .literal, .FILENAME {
-	color: #007a00;
-	font-family: monospace;
-}
-
-
-.GUIMENU, .GUIMENUITEM, .GUISUBMENU,
-.GUILABEL, .INTERFACE,
-.SHORTCUT, .SHORTCUT .KEYCAP {
-	font-weight: bold;
-}
-
-.GUIBUTTON {
-	background-color: #CFCFCF;
-	padding: 2px;
-}
-
-.ACCEL {
-	background-color: #F0F0F0;
-	text-decoration: underline;
-} 
-
-.PROGRAMLISTING {
-	padding: 1ex;
-	background-color: #eee;
-	border: 1px solid #ccc;
-}
-
-@media screen {  /* hide from IE3 */:/
-	a[href]:hover { background: #ffa }
-}
-
-.NOTE, .SCREEN {
-	color: #222;
-	background: #fff;
-	border: 1px solid #ccc;
-	padding: 0.4em 0.4em;
-}
-
-.SCREEN {
-        background: #eee;
-}
-
-.TIP {
-	color: #004F00;
-	background: #fff;
-	border: 1px solid green;
-	padding: 0.4em 0.4em;
-}
-
-table.CDH {
-	padding:0px 0px 8px
-}
-
-table.CDH th {
-	background: #EDF2F9;
-	border-right: 0.5pt solid #adb6cd;
-	border-bottom: 0.5pt solid #adb6cd;
-        padding:2px 8px 1px;
-}
-
-table.CDH td {
-	font-size: 12px;
-}
-
-div.breadcrumbs {
-	border-top: 2px solid #0068BB;
-        background: #EDF2F9;
-	font-style: bold;
-	font-size: small;
-	padding: 0.6em 0.6em;
-}
-
-span.breadcrumb-node {
-	font-style: italic;
-}
-
-/* Match the the breadcrumbs at the top */
-div.footer-text {
-	border-bottom: 2px solid #0068BB;
-        background: #EDF2F9;
-        font-size: small;
-        padding: 0.2em 0.2em;
-	text-align: center;
-}
-
-.IMPORTANT, .caution {
-	font-style:italic;
-	border: 1px solid #a00;
-	border-left: 12px solid #c00;
-	padding: 0.1em 1em;
-}
-
-.WARNING {
-	color: #9F1313;
-	background: #fff;
-	border: 1px solid #e59595;
-	padding: 0.4em 0.4em;
-	width: 85%;
-}
-
-.EXAMPLE {
-	width: 100%;
-}
-
-.tip, .note, .caution {
-	margin:4px 0 4px;
-}
-
-img {
-	border: 0;
-}
diff --git a/docs/DeveloperGuide/footer.xsl b/docs/DeveloperGuide/footer.xsl
deleted file mode 100644
index 98582c0..0000000
--- a/docs/DeveloperGuide/footer.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:template name="user.footer.content">
-
-  <script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script>
-  <script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script>
-
-  <div class="footer-text">
-  <span align="center"><a href="index.html"><img src="images/home.png"
-      alt="Documentation Home" /></a></span>
-  <br/>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/DeveloperGuide/header.xsl b/docs/DeveloperGuide/header.xsl
deleted file mode 100644
index 7c74909..0000000
--- a/docs/DeveloperGuide/header.xsl
+++ /dev/null
@@ -1,41 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:import href="breadcrumbs.xsl"/>
-
-<xsl:template name="user.head.content">
-</xsl:template>
-
-<xsl:template name="user.header.content">
-
-    <div style="clear:both; margin-bottom: 4px" />
-    <div align="center">
-      <a href="index.html"><img src="images/home.png"
-          alt="Documentation Home" /></a>
-    </div>
-    <span class="breadcrumbs">
-    <xsl:call-template name="breadcrumbs"/>
-    </span>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/DeveloperGuide/html.xsl b/docs/DeveloperGuide/html.xsl
deleted file mode 100644
index 2617dd4..0000000
--- a/docs/DeveloperGuide/html.xsl
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version='1.0'?>
-
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- $FreeBSD: doc/share/xsl/freebsd-html.xsl,v 1.1 2003/01/03 05:06:14 trhodes Exp $ -->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                version='1.0'
-                xmlns="http://www.w3.org/TR/xhtml1/transitional"
-                exclude-result-prefixes="#default">
-
-  <xsl:import href="header.xsl"/>
-  <xsl:import href="footer.xsl"/>
-
-  <xsl:template name="body.attributes" />
-
-  <xsl:param name="html.stylesheet" select="'docbook.css'"/>
-  <xsl:param name="use.id.as.filename" select="1"/>
-  <xsl:param name="generate.legalnotice.link" select="'1'"/>
-  <xsl:param name="generate.section.toc.level" select="100"></xsl:param>
-
-  <xsl:param name="toc.section.depth" select="10"/>
-  <xsl:param name="section.autolabel" select="1"/>
-  <xsl:param name="section.label.includes.component.label" select="1"/>
-  <xsl:param name="chunk.section.depth" select="100"></xsl:param>
-  <xsl:param name="chunk.first.sections" select="1"></xsl:param>
-  <xsl:param name="navig.showtitles" select="1"></xsl:param>
-
-  <xsl:param name="admon.graphics" select="1"></xsl:param>
-  <xsl:param name="admon.graphics.extension">.png</xsl:param>
-  <xsl:param name="admon.graphics.path">images/</xsl:param>
-
-  <xsl:param name="navig.graphics" select="1"></xsl:param>
-  <xsl:param name="navig.graphics.extension">.png</xsl:param>
-  <xsl:param name="navig.graphics.path">images/</xsl:param>
-
-  <xsl:param name="header.rule" select="0"></xsl:param>
-  <xsl:param name="footer.rule" select="0"></xsl:param>
-  <xsl:param name="suppress.header.navigation" select="1"></xsl:param>
-
-  <xsl:param name="generate.index" select="1"></xsl:param>
-
-  <xsl:param name="spacing.paras" select="0"></xsl:param>
-  <xsl:param name="html.cleanup" select="1"></xsl:param>
-
-  <xsl:param name="table.borders.with.css" select="1"></xsl:param>
-  <!-- xsl:param name="id.warnings" select="1"></xsl:param -->
-
-  <xsl:param name="generate.toc">
-appendix  toc,title
-article   toc,title,figure,equation
-book      toc,title,figure,example,equation
-chapter   toc,title
-part      toc,title
-preface   toc,title
-qandadiv  toc
-qandaset  toc
-reference toc,title
-sect1     toc
-sect2     toc
-sect3     toc
-sect4     toc
-sect5     toc
-section   toc
-set       toc,title
-   </xsl:param>
-
-</xsl:stylesheet>
diff --git a/docs/DeveloperGuide/images/README b/docs/DeveloperGuide/images/README
deleted file mode 100644
index f12b2a7..0000000
--- a/docs/DeveloperGuide/images/README
+++ /dev/null
@@ -1,5 +0,0 @@
-Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook
-icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency
-from the Jimmac icons to get round MS IE and FOP PNG incompatibilies.
-
-Stuart Rackham
diff --git a/docs/DeveloperGuide/images/callouts/1.png b/docs/DeveloperGuide/images/callouts/1.png
deleted file mode 100644
index 7d473430b7bec514f7de12f5769fe7c5859e8c5d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/10.png b/docs/DeveloperGuide/images/callouts/10.png
deleted file mode 100644
index 997bbc8246a316e040e0804174ba260e219d7d33..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/11.png b/docs/DeveloperGuide/images/callouts/11.png
deleted file mode 100644
index ce47dac3f52ac49017749a3fea53db57d006993c..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/12.png b/docs/DeveloperGuide/images/callouts/12.png
deleted file mode 100644
index 31daf4e2f25b6712499ee32de9c2e3b050b691ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/13.png b/docs/DeveloperGuide/images/callouts/13.png
deleted file mode 100644
index 14021a89c2ed3d4881afea6e3a315bce4f95efce..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/14.png b/docs/DeveloperGuide/images/callouts/14.png
deleted file mode 100644
index 64014b75fe2e84d45ed861974c72462727979360..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/15.png b/docs/DeveloperGuide/images/callouts/15.png
deleted file mode 100644
index 0d65765fcf13dcfd87914744dec8bda115e4adf1..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/2.png b/docs/DeveloperGuide/images/callouts/2.png
deleted file mode 100644
index 5d09341b2f6d2ea2d1d5dad5d980f14b4b05dfd2..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/3.png b/docs/DeveloperGuide/images/callouts/3.png
deleted file mode 100644
index ef7b70047158970cf4e09f1bab2954d39c2d596b..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/4.png b/docs/DeveloperGuide/images/callouts/4.png
deleted file mode 100644
index adb8364eb5d21ecdd4086e16110b62ddcb42aa4a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/5.png b/docs/DeveloperGuide/images/callouts/5.png
deleted file mode 100644
index 4d7eb460021e845981861d77614539314f553993..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/6.png b/docs/DeveloperGuide/images/callouts/6.png
deleted file mode 100644
index 0ba694af6c07d947d219b45a629bd32c60a0f5fe..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/7.png b/docs/DeveloperGuide/images/callouts/7.png
deleted file mode 100644
index 472e96f8ac36862c5645732f2fff19d06ee11a8e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/8.png b/docs/DeveloperGuide/images/callouts/8.png
deleted file mode 100644
index 5e60973c213b37df93666c5a00724f34493974ae..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/callouts/9.png b/docs/DeveloperGuide/images/callouts/9.png
deleted file mode 100644
index a0676d26cc2ff1de12c4ecdeefb44a0d71bc6bde..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/caution.png b/docs/DeveloperGuide/images/caution.png
deleted file mode 100644
index cb9d5ea0df3fb7abe744d6ef885847c13b786d4d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/example.png b/docs/DeveloperGuide/images/example.png
deleted file mode 100644
index bba1c0010ddbb148a93e8a442047ca6a68b5ef1f..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/home.png b/docs/DeveloperGuide/images/home.png
deleted file mode 100644
index 37a5231bacc905886c9351a291e0d5eed68dd60e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/important.png b/docs/DeveloperGuide/images/important.png
deleted file mode 100644
index 1096c232959e8cb5c2977b5eb44c1b67173cb422..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/next.png b/docs/DeveloperGuide/images/next.png
deleted file mode 100644
index 64e126bdda3f2e611199744494367ad0c1a1f9ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/note.png b/docs/DeveloperGuide/images/note.png
deleted file mode 100644
index 841820f7c4fb274fc45fae0a0a526a68eb31d85e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/prev.png b/docs/DeveloperGuide/images/prev.png
deleted file mode 100644
index 3e8f12fe24c6e042bc7ef4badb15449f71529ace..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/tip.png b/docs/DeveloperGuide/images/tip.png
deleted file mode 100644
index a3a029d89891bff666fbdc1a082ae4a7951b5abf..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/up.png b/docs/DeveloperGuide/images/up.png
deleted file mode 100644
index 2db1ce62fa575fd990108b15b0de18a64cae19ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/images/warning.png b/docs/DeveloperGuide/images/warning.png
deleted file mode 100644
index 0b0c419df29b7e71ef14ecf5970b2abcd6d5ea01..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/DeveloperGuide/index.html b/docs/DeveloperGuide/index.html
deleted file mode 100644
index d3d60ef..0000000
--- a/docs/DeveloperGuide/index.html
+++ /dev/null
@@ -1,28 +0,0 @@
-<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>Flume Developer Guide</title><link rel="stylesheet" href="docbook.css" type="text/css"><meta name="generator" content="DocBook XSL Stylesheets V1.75.2"></head><body><div style="clear:both; margin-bottom: 4px"></div><div align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></div><span class="breadcrumbs"><div class="breadcrumbs"><span class="breadcrumb-node">Flume Developer Guide</span></div></span><div lang="en" class="article" title="Flume Developer Guide"><div class="titlepage"><div><div><h2 class="title"><a name="id538023"></a>Flume Developer Guide</h2></div><div><div class="author"><h3 class="author"><span class="firstname">flume-dev@cloudera.org</span></h3></div></div><div><div class="revhistory"><table border="1" width="100%" summary="Revision history"><tr><th align="left" valign="top" colspan="3"><b>Revision History</b></th></tr><tr><td align="left">Revision 0.9.3-SNAPSHOT</td><td align="left">November 16 2010</td><td align="left">F</td></tr></table></div></div></div><hr></div><div class="toc"><p><b>Table of Contents</b></p><dl><dt><span class="section"><a href="#_logging">1. Logging</a></span></dt><dd><dl><dt><span class="section"><a href="#_setting_up_logging">1.1. Setting Up Logging</a></span></dt><dt><span class="section"><a href="#_logging_exceptions">1.2. Logging Exceptions</a></span></dt><dt><span class="section"><a href="#_logging_formatting_and_performance_notes">1.3. Logging Formatting and Performance Notes</a></span></dt></dl></dd><dt><span class="section"><a href="#_writing_documentation">2. Writing Documentation</a></span></dt></dl></div><div class="section" title="1.�Logging"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_logging"></a>1.�Logging</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_setting_up_logging">1.1. Setting Up Logging</a></span></dt><dt><span class="section"><a href="#_logging_exceptions">1.2. Logging Exceptions</a></span></dt><dt><span class="section"><a href="#_logging_formatting_and_performance_notes">1.3. Logging Formatting and Performance Notes</a></span></dt></dl></div><p>Logging is implemented using the <a class="ulink" href="http://www.slf4j.org/" target="_top">SLF4J API</a>.</p><p>Be sure to read the <a class="ulink" href="http://www.slf4j.org/docs.html" target="_top">SLF4J documentation</a>.</p><div class="section" title="1.1.�Setting Up Logging"><div class="titlepage"><div><div><h3 class="title"><a name="_setting_up_logging"></a>1.1.�Setting Up Logging</h3></div></div></div><p>Typically each class has its own logger. We declare these as follows:</p><pre class="programlisting">private static final Logger LOG = LoggerFactory.getLogger(MyClass.class);</pre><p>Things to note here:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-We make all loggers <code class="literal">private static final</code> (and in that order).
-</li><li class="listitem">
-We name all of our loggers <code class="literal">LOG</code>.
-</li><li class="listitem">
-We just pass the class, we don&#8217;t call <code class="literal">.getName()</code> on the class.
-</li></ul></div></div><div class="section" title="1.2.�Logging Exceptions"><div class="titlepage"><div><div><h3 class="title"><a name="_logging_exceptions"></a>1.2.�Logging Exceptions</h3></div></div></div><p>We are using the 1.6 version of SLF4J, so we get to take advantage of
-the <a class="ulink" href="http://www.slf4j.org/faq.html#paramException" target="_top">improved support</a> for
-logging exceptions.</p></div><div class="section" title="1.3.�Logging Formatting and Performance Notes"><div class="titlepage"><div><div><h3 class="title"><a name="_logging_formatting_and_performance_notes"></a>1.3.�Logging Formatting and Performance Notes</h3></div></div></div><p>For a good discussion of this, see the
-<a class="ulink" href="http://www.slf4j.org/faq.html#logging_performance" target="_top">relevant section</a>
-in the SLF4J FAQ.</p></div></div><div class="section" title="2.�Writing Documentation"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_writing_documentation"></a>2.�Writing Documentation</h2></div></div></div><p>Documentation is written using <a class="ulink" href="http://www.methods.co.nz/asciidoc/" target="_top">AsciiDoc</a>.</p><pre class="literallayout">     ______
-    / ___//_  ______  ____
-   / /_/ / / / /    \/ __/
-  / __/ / /_/ / / / / __/
- / / /_/\____/_/_/_/\__/
-/_/ Distributed Log Collection.</pre></div></div><script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script><script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script><div class="footer-text"><span align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></span><br>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div></body></html>
diff --git a/docs/DeveloperGuide/index.txt b/docs/DeveloperGuide/index.txt
deleted file mode 100644
index 04ff317..0000000
--- a/docs/DeveloperGuide/index.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-
-////
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-////
-
-Flume Documentation (v{revnumber})
-==================================
-
-The following pages are the documentation for Flume v{revnumber}:
-
-- link:UserGuide/index.html[Flume User Guide]
-- link:DeveloperGuide/index.html[Flume Developer Guide]
-- link:Cookbook/index.html[Flume Cookbook]
-
diff --git a/docs/UserGuide/architecture.png b/docs/UserGuide/architecture.png
deleted file mode 100644
index 816a72d5b3e249d696e11fd1d16255e9ecd1fe0f..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/breadcrumbs.xsl b/docs/UserGuide/breadcrumbs.xsl
deleted file mode 100644
index 98d70ba..0000000
--- a/docs/UserGuide/breadcrumbs.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<xsl:template name="breadcrumbs">
-  <xsl:param name="this.node" select="."/>
-  <div class="breadcrumbs">
-    <xsl:for-each select="$this.node/ancestor::*">
-      <span class="breadcrumb-link">
-        <a>
-          <xsl:attribute name="href">
-            <xsl:call-template name="href.target">
-              <xsl:with-param name="object" select="."/>
-              <xsl:with-param name="context" select="$this.node"/>
-            </xsl:call-template>
-          </xsl:attribute>
-          <xsl:apply-templates select="." mode="title.markup"/>
-        </a>
-      </span>
-      <xsl:text> &gt; </xsl:text>
-    </xsl:for-each>
-    <!-- And display the current node, but not as a link -->
-    <span class="breadcrumb-node">
-      <xsl:apply-templates select="$this.node" mode="title.markup"/>
-    </span>
-  </div>
-</xsl:template>
-
-</xsl:stylesheet>
diff --git a/docs/UserGuide/docbook.css b/docs/UserGuide/docbook.css
deleted file mode 100644
index a622c9f..0000000
--- a/docs/UserGuide/docbook.css
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- *
- *
- * This file was provided to this project under these terms:
- *
- * Copyright (c) 2001, 2003 The FreeBSD Documentation Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD: doc/share/misc/docbook.css,v 1.14 2008/11/21 07:28:34 keramida Exp $
- */
-
-BODY ADDRESS {
-	line-height: 1.3;
-	margin: .6em 0;
-}
-
-BODY BLOCKQUOTE {
-	margin-top: .75em;
-	line-height: 1.5;
-	margin-bottom: .75em;
-}
-
-HTML BODY {
-	margin: 1em 4% 1em 4%;
-	line-height: 1.2;
-	font-family: verdana;
-}
-
-a {
-    text-decoration: none;
-    color: #1c84bf;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-a:visited {
-    color: #597e93;
-}
-
-div.titlepage .title {
-	font-size: 26px;
-}
-
-.LEGALNOTICE {
-	font-size: small;
-	font-variant: small-caps;
-}
-
-BODY DIV {
-	margin: 0;
-}
-
-DL {
-	margin: .8em 0;
-	line-height: 1.2;
-}
-
-BODY FORM {
-	margin: .6em 0;
-}
-
-H1, H2, H3, H4, H5, H6,
-DIV.EXAMPLE P B,
-.QUESTION,
-DIV.TABLE P B,
-DIV.PROCEDURE P B {
-	color: #148BCF;
-}
-
-BODY H1, BODY H2, BODY H3, BODY H4, BODY H5, BODY H6 {
-	line-height: 1.3;
-	margin-left: 0;
-}
-
-BODY H1, H2, H3, H4, H5, H6 {
-	margin: 6px 0 0 0;
-}
-
-BODY HR {
-	border-width: 0 0 1px 0;
-	border-style: solid;
-	border-color: #0068BB;
-}
-
-BODY IMG.NAVHEADER {
-	margin: 0 0 0 -4%;
-}
-
-OL {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-BODY PRE {
-	margin: .75em 0;
-	line-height: 1.0;
-	font-family: monospace;
-}
-
-BODY TD, BODY TH {
-	line-height: 1.2;
-}
-
-UL, BODY DIR, BODY MENU {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-HTML {
-	margin: 0; 
-	padding: 0;
-}
-
-BODY P B.APPLICATION {
-	color: #000000;
-}
-
-.navfooter td, .literal, .FILENAME {
-	color: #007a00;
-	font-family: monospace;
-}
-
-
-.GUIMENU, .GUIMENUITEM, .GUISUBMENU,
-.GUILABEL, .INTERFACE,
-.SHORTCUT, .SHORTCUT .KEYCAP {
-	font-weight: bold;
-}
-
-.GUIBUTTON {
-	background-color: #CFCFCF;
-	padding: 2px;
-}
-
-.ACCEL {
-	background-color: #F0F0F0;
-	text-decoration: underline;
-} 
-
-.PROGRAMLISTING {
-	padding: 1ex;
-	background-color: #eee;
-	border: 1px solid #ccc;
-}
-
-@media screen {  /* hide from IE3 */:/
-	a[href]:hover { background: #ffa }
-}
-
-.NOTE, .SCREEN {
-	color: #222;
-	background: #fff;
-	border: 1px solid #ccc;
-	padding: 0.4em 0.4em;
-}
-
-.SCREEN {
-        background: #eee;
-}
-
-.TIP {
-	color: #004F00;
-	background: #fff;
-	border: 1px solid green;
-	padding: 0.4em 0.4em;
-}
-
-table.CDH {
-	padding:0px 0px 8px
-}
-
-table.CDH th {
-	background: #EDF2F9;
-	border-right: 0.5pt solid #adb6cd;
-	border-bottom: 0.5pt solid #adb6cd;
-        padding:2px 8px 1px;
-}
-
-table.CDH td {
-	font-size: 12px;
-}
-
-div.breadcrumbs {
-	border-top: 2px solid #0068BB;
-        background: #EDF2F9;
-	font-style: bold;
-	font-size: small;
-	padding: 0.6em 0.6em;
-}
-
-span.breadcrumb-node {
-	font-style: italic;
-}
-
-/* Match the the breadcrumbs at the top */
-div.footer-text {
-	border-bottom: 2px solid #0068BB;
-        background: #EDF2F9;
-        font-size: small;
-        padding: 0.2em 0.2em;
-	text-align: center;
-}
-
-.IMPORTANT, .caution {
-	font-style:italic;
-	border: 1px solid #a00;
-	border-left: 12px solid #c00;
-	padding: 0.1em 1em;
-}
-
-.WARNING {
-	color: #9F1313;
-	background: #fff;
-	border: 1px solid #e59595;
-	padding: 0.4em 0.4em;
-	width: 85%;
-}
-
-.EXAMPLE {
-	width: 100%;
-}
-
-.tip, .note, .caution {
-	margin:4px 0 4px;
-}
-
-img {
-	border: 0;
-}
diff --git a/docs/UserGuide/failoverCollector.png b/docs/UserGuide/failoverCollector.png
deleted file mode 100644
index 0c23c2bd25c9cf6a4800232b089929788dac3949..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/footer.xsl b/docs/UserGuide/footer.xsl
deleted file mode 100644
index 98582c0..0000000
--- a/docs/UserGuide/footer.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:template name="user.footer.content">
-
-  <script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script>
-  <script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script>
-
-  <div class="footer-text">
-  <span align="center"><a href="index.html"><img src="images/home.png"
-      alt="Documentation Home" /></a></span>
-  <br/>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/UserGuide/header.xsl b/docs/UserGuide/header.xsl
deleted file mode 100644
index 7c74909..0000000
--- a/docs/UserGuide/header.xsl
+++ /dev/null
@@ -1,41 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:import href="breadcrumbs.xsl"/>
-
-<xsl:template name="user.head.content">
-</xsl:template>
-
-<xsl:template name="user.header.content">
-
-    <div style="clear:both; margin-bottom: 4px" />
-    <div align="center">
-      <a href="index.html"><img src="images/home.png"
-          alt="Documentation Home" /></a>
-    </div>
-    <span class="breadcrumbs">
-    <xsl:call-template name="breadcrumbs"/>
-    </span>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/UserGuide/html.xsl b/docs/UserGuide/html.xsl
deleted file mode 100644
index 2617dd4..0000000
--- a/docs/UserGuide/html.xsl
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version='1.0'?>
-
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- $FreeBSD: doc/share/xsl/freebsd-html.xsl,v 1.1 2003/01/03 05:06:14 trhodes Exp $ -->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                version='1.0'
-                xmlns="http://www.w3.org/TR/xhtml1/transitional"
-                exclude-result-prefixes="#default">
-
-  <xsl:import href="header.xsl"/>
-  <xsl:import href="footer.xsl"/>
-
-  <xsl:template name="body.attributes" />
-
-  <xsl:param name="html.stylesheet" select="'docbook.css'"/>
-  <xsl:param name="use.id.as.filename" select="1"/>
-  <xsl:param name="generate.legalnotice.link" select="'1'"/>
-  <xsl:param name="generate.section.toc.level" select="100"></xsl:param>
-
-  <xsl:param name="toc.section.depth" select="10"/>
-  <xsl:param name="section.autolabel" select="1"/>
-  <xsl:param name="section.label.includes.component.label" select="1"/>
-  <xsl:param name="chunk.section.depth" select="100"></xsl:param>
-  <xsl:param name="chunk.first.sections" select="1"></xsl:param>
-  <xsl:param name="navig.showtitles" select="1"></xsl:param>
-
-  <xsl:param name="admon.graphics" select="1"></xsl:param>
-  <xsl:param name="admon.graphics.extension">.png</xsl:param>
-  <xsl:param name="admon.graphics.path">images/</xsl:param>
-
-  <xsl:param name="navig.graphics" select="1"></xsl:param>
-  <xsl:param name="navig.graphics.extension">.png</xsl:param>
-  <xsl:param name="navig.graphics.path">images/</xsl:param>
-
-  <xsl:param name="header.rule" select="0"></xsl:param>
-  <xsl:param name="footer.rule" select="0"></xsl:param>
-  <xsl:param name="suppress.header.navigation" select="1"></xsl:param>
-
-  <xsl:param name="generate.index" select="1"></xsl:param>
-
-  <xsl:param name="spacing.paras" select="0"></xsl:param>
-  <xsl:param name="html.cleanup" select="1"></xsl:param>
-
-  <xsl:param name="table.borders.with.css" select="1"></xsl:param>
-  <!-- xsl:param name="id.warnings" select="1"></xsl:param -->
-
-  <xsl:param name="generate.toc">
-appendix  toc,title
-article   toc,title,figure,equation
-book      toc,title,figure,example,equation
-chapter   toc,title
-part      toc,title
-preface   toc,title
-qandadiv  toc
-qandaset  toc
-reference toc,title
-sect1     toc
-sect2     toc
-sect3     toc
-sect4     toc
-sect5     toc
-section   toc
-set       toc,title
-   </xsl:param>
-
-</xsl:stylesheet>
diff --git a/docs/UserGuide/images/README b/docs/UserGuide/images/README
deleted file mode 100644
index f12b2a7..0000000
--- a/docs/UserGuide/images/README
+++ /dev/null
@@ -1,5 +0,0 @@
-Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook
-icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency
-from the Jimmac icons to get round MS IE and FOP PNG incompatibilies.
-
-Stuart Rackham
diff --git a/docs/UserGuide/images/callouts/1.png b/docs/UserGuide/images/callouts/1.png
deleted file mode 100644
index 7d473430b7bec514f7de12f5769fe7c5859e8c5d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/10.png b/docs/UserGuide/images/callouts/10.png
deleted file mode 100644
index 997bbc8246a316e040e0804174ba260e219d7d33..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/11.png b/docs/UserGuide/images/callouts/11.png
deleted file mode 100644
index ce47dac3f52ac49017749a3fea53db57d006993c..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/12.png b/docs/UserGuide/images/callouts/12.png
deleted file mode 100644
index 31daf4e2f25b6712499ee32de9c2e3b050b691ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/13.png b/docs/UserGuide/images/callouts/13.png
deleted file mode 100644
index 14021a89c2ed3d4881afea6e3a315bce4f95efce..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/14.png b/docs/UserGuide/images/callouts/14.png
deleted file mode 100644
index 64014b75fe2e84d45ed861974c72462727979360..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/15.png b/docs/UserGuide/images/callouts/15.png
deleted file mode 100644
index 0d65765fcf13dcfd87914744dec8bda115e4adf1..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/2.png b/docs/UserGuide/images/callouts/2.png
deleted file mode 100644
index 5d09341b2f6d2ea2d1d5dad5d980f14b4b05dfd2..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/3.png b/docs/UserGuide/images/callouts/3.png
deleted file mode 100644
index ef7b70047158970cf4e09f1bab2954d39c2d596b..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/4.png b/docs/UserGuide/images/callouts/4.png
deleted file mode 100644
index adb8364eb5d21ecdd4086e16110b62ddcb42aa4a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/5.png b/docs/UserGuide/images/callouts/5.png
deleted file mode 100644
index 4d7eb460021e845981861d77614539314f553993..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/6.png b/docs/UserGuide/images/callouts/6.png
deleted file mode 100644
index 0ba694af6c07d947d219b45a629bd32c60a0f5fe..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/7.png b/docs/UserGuide/images/callouts/7.png
deleted file mode 100644
index 472e96f8ac36862c5645732f2fff19d06ee11a8e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/8.png b/docs/UserGuide/images/callouts/8.png
deleted file mode 100644
index 5e60973c213b37df93666c5a00724f34493974ae..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/callouts/9.png b/docs/UserGuide/images/callouts/9.png
deleted file mode 100644
index a0676d26cc2ff1de12c4ecdeefb44a0d71bc6bde..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/caution.png b/docs/UserGuide/images/caution.png
deleted file mode 100644
index cb9d5ea0df3fb7abe744d6ef885847c13b786d4d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/example.png b/docs/UserGuide/images/example.png
deleted file mode 100644
index bba1c0010ddbb148a93e8a442047ca6a68b5ef1f..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/home.png b/docs/UserGuide/images/home.png
deleted file mode 100644
index 37a5231bacc905886c9351a291e0d5eed68dd60e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/important.png b/docs/UserGuide/images/important.png
deleted file mode 100644
index 1096c232959e8cb5c2977b5eb44c1b67173cb422..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/next.png b/docs/UserGuide/images/next.png
deleted file mode 100644
index 64e126bdda3f2e611199744494367ad0c1a1f9ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/note.png b/docs/UserGuide/images/note.png
deleted file mode 100644
index 841820f7c4fb274fc45fae0a0a526a68eb31d85e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/prev.png b/docs/UserGuide/images/prev.png
deleted file mode 100644
index 3e8f12fe24c6e042bc7ef4badb15449f71529ace..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/tip.png b/docs/UserGuide/images/tip.png
deleted file mode 100644
index a3a029d89891bff666fbdc1a082ae4a7951b5abf..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/up.png b/docs/UserGuide/images/up.png
deleted file mode 100644
index 2db1ce62fa575fd990108b15b0de18a64cae19ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/images/warning.png b/docs/UserGuide/images/warning.png
deleted file mode 100644
index 0b0c419df29b7e71ef14ecf5970b2abcd6d5ea01..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/index.html b/docs/UserGuide/index.html
deleted file mode 100644
index a9db411..0000000
--- a/docs/UserGuide/index.html
+++ /dev/null
@@ -1,3113 +0,0 @@
-<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>Flume User Guide</title><link rel="stylesheet" href="docbook.css" type="text/css"><meta name="generator" content="DocBook XSL Stylesheets V1.75.2"></head><body><div style="clear:both; margin-bottom: 4px"></div><div align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></div><span class="breadcrumbs"><div class="breadcrumbs"><span class="breadcrumb-node">Flume User Guide</span></div></span><div lang="en" class="article" title="Flume User Guide"><div class="titlepage"><div><div><h2 class="title"><a name="id402441"></a>Flume User Guide</h2></div><div><div class="author"><h3 class="author"><span class="firstname">flume-dev@cloudera.org</span></h3></div></div><div><div class="revhistory"><table border="1" width="100%" summary="Revision history"><tr><th align="left" valign="top" colspan="3"><b>Revision History</b></th></tr><tr><td align="left">Revision 0.9.3-SNAPSHOT</td><td align="left">November 16 2010</td><td align="left">F</td></tr></table></div></div></div><hr></div><div class="toc"><p><b>Table of Contents</b></p><dl><dt><span class="section"><a href="#Introduction">1. Introduction</a></span></dt><dd><dl><dt><span class="section"><a href="#_architecture">1.1. Architecture</a></span></dt><dt><span class="section"><a href="#_reliability">1.2. Reliability</a></span></dt><dt><span class="section"><a href="#_scalability">1.3. Scalability</a></span></dt><dt><span class="section"><a href="#_manageability">1.4. Manageability</a></span></dt><dt><span class="section"><a href="#_extensibility">1.5. Extensibility</a></span></dt><dt><span class="section"><a href="#_section_summary">1.6. Section summary</a></span></dt></dl></dd><dt><span class="section"><a href="#Quickstart">2. Flume Single Node Quick Start</a></span></dt><dd><dl><dt><span class="section"><a href="#_sources_and_the_literal_dump_literal_command">2.1. Sources and the <code class="literal">dump</code> command</a></span></dt><dd><dl><dt><span class="section"><a href="#_reading_from_a_text_file_literal_text_literal">2.1.1. Reading from a text file, <code class="literal">text</code></a></span></dt><dt><span class="section"><a href="#_tailing_a_file_name_literal_tail_literal_and_literal_multitail_literal">2.1.2. Tailing a file name, <code class="literal">tail</code> and <code class="literal">multitail</code></a></span></dt><dt><span class="section"><a href="#_synthetic_sources_literal_synth_literal">2.1.3. Synthetic sources, <code class="literal">synth</code></a></span></dt><dt><span class="section"><a href="#_syslog_as_a_source_literal_syslogudp_literal_and_literal_syslogtcp_literal">2.1.4. Syslog as a source, <code class="literal">syslogUdp</code> and <code class="literal">syslogTcp</code></a></span></dt></dl></dd><dt><span class="section"><a href="#_anatomy_of_an_event">2.2. Anatomy of an Event</a></span></dt><dt><span class="section"><a href="#_section_summary_2">2.3. Section Summary</a></span></dt></dl></dd><dt><span class="section"><a href="#_pseudo_distributed_mode">3. Pseudo-distributed Mode</a></span></dt><dd><dl><dt><span class="section"><a href="#_starting_pseudo_distributed_flume_daemons">3.1. Starting Pseudo-distributed Flume Daemons</a></span></dt><dd><dl><dt><span class="section"><a href="#_the_master">3.1.1. The Master</a></span></dt><dt><span class="section"><a href="#_the_flume_node">3.1.2. The Flume Node</a></span></dt></dl></dd><dt><span class="section"><a href="#_configuring_a_node_via_the_master">3.2. Configuring a Node via the Master</a></span></dt><dt><span class="section"><a href="#_introducing_sinks">3.3. Introducing Sinks</a></span></dt><dt><span class="section"><a href="#_aggregated_configurations">3.4. Aggregated Configurations</a></span></dt><dt><span class="section"><a href="#_tiering_flume_nodes_agents_and_collectors">3.5. Tiering Flume Nodes: Agents and Collectors</a></span></dt><dt><span class="section"><a href="#_section_summary_3">3.6. Section Summary</a></span></dt></dl></dd><dt><span class="section"><a href="#_fully_distributed_mode">4. Fully-distributed Mode</a></span></dt><dd><dl><dt><span class="section"><a href="#_static_configuration_files">4.1. Static Configuration Files</a></span></dt><dd><dl><dt><span class="section"><a href="#_using_default_values">4.1.1. Using Default Values</a></span></dt></dl></dd><dt><span class="section"><a href="#_multiple_collectors">4.2. Multiple Collectors</a></span></dt><dd><dl><dt><span class="section"><a href="#_partitioning_agents_across_multiple_collectors">4.2.1. Partitioning Agents across Multiple Collectors</a></span></dt><dt><span class="section"><a href="#_manually_specifying_failover_chains">4.2.2. Manually Specifying Failover Chains</a></span></dt><dt><span class="section"><a href="#_automatic_failover_chains">4.2.3. Automatic Failover Chains</a></span></dt></dl></dd><dt><span class="section"><a href="#_logical_configurations">4.3. Logical Configurations</a></span></dt><dd><dl><dt><span class="section"><a href="#_logical_nodes">4.3.1. Logical Nodes</a></span></dt><dt><span class="section"><a href="#_logical_sources_and_logical_sinks">4.3.2. Logical Sources and Logical Sinks</a></span></dt><dt><span class="section"><a href="#_flow_isolation">4.3.3. Flow Isolation</a></span></dt><dt><span class="section"><a href="#_section_summary_4">4.3.4. Section Summary</a></span></dt></dl></dd><dt><span class="section"><a href="#_multiple_masters">4.4. Multiple Masters</a></span></dt><dd><dl><dt><span class="section"><a href="#_standalone_mode_compared_to_distributed_mode">4.4.1. Standalone Mode Compared to Distributed Mode</a></span></dt><dt><span class="section"><a href="#_running_in_standalone_mode">4.4.2. Running in Standalone Mode</a></span></dt><dt><span class="section"><a href="#_running_in_distributed_mode">4.4.3. Running in Distributed Mode</a></span></dt><dt><span class="section"><a href="#_configuration_stores">4.4.4. Configuration Stores</a></span></dt><dt><span class="section"><a href="#_which_configuration_store_should_i_use">4.4.5. Which Configuration Store Should I Use?</a></span></dt><dt><span class="section"><a href="#_configuring_the_zbcs">4.4.6. Configuring the ZBCS</a></span></dt><dt><span class="section"><a href="#_gossip_in_distributed_mode">4.4.7. Gossip in Distributed Mode</a></span></dt><dt><span class="section"><a href="#_diagrams_how_the_masters_and_nodes_talk_to_each_other">4.4.8. Diagrams: How the Masters and Nodes talk to each other</a></span></dt><dt><span class="section"><a href="#_configuring_flume_nodes_to_connect_to_multiple_master_servers">4.4.9. Configuring Flume Nodes to Connect to Multiple Master Servers</a></span></dt></dl></dd><dt><span class="section"><a href="#_external_zookeeper_cluster">4.5. External ZooKeeper Cluster</a></span></dt><dt><span class="section"><a href="#_section_summary_5">4.6. Section Summary</a></span></dt></dl></dd><dt><span class="section"><a href="#_integrating_flume_with_your_data_sources">5. Integrating Flume with your Data Sources</a></span></dt><dd><dl><dt><span class="section"><a href="#_push_sources">5.1. Push Sources</a></span></dt><dt><span class="section"><a href="#_polling_sources">5.2. Polling Sources</a></span></dt><dt><span class="section"><a href="#_embedding_sources">5.3. Embedding Sources</a></span></dt><dt><span class="section"><a href="#_logging_via_log4j_directly">5.4. Logging via log4j Directly</a></span></dt><dd><dl><dt><span class="section"><a href="#_example_of_logging_hadoop_jobs">5.4.1. Example of Logging Hadoop Jobs</a></span></dt><dt><span class="section"><a href="#_logging_hadoop_daemons">5.4.2. Logging Hadoop Daemons</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="#_using_data_collected_by_flume">6. Using Data Collected by Flume</a></span></dt><dd><dl><dt><span class="section"><a href="#_the_data_model_of_a_flume_event">6.1. The Data Model of a Flume Event</a></span></dt><dt><span class="section"><a href="#_output_bucketing">6.2. Output Bucketing</a></span></dt><dt><span class="section"><a href="#_output_format">6.3. Output Format</a></span></dt><dt><span class="section"><a href="#_small_files_compared_to_high_latency">6.4. Small Files Compared to High Latency</a></span></dt></dl></dd><dt><span class="section"><a href="#_compression_for_files_written_to_hdfs">7. Compression for files written to HDFS.</a></span></dt><dt><span class="section"><a href="#_advanced_flume_usage">8. Advanced Flume Usage</a></span></dt><dd><dl><dt><span class="section"><a href="#_the_flume_command_shell">8.1. The Flume Command Shell</a></span></dt><dd><dl><dt><span class="section"><a href="#_using_the_flume_command_shell">8.1.1. Using the Flume Command Shell</a></span></dt></dl></dd><dt><span class="section"><a href="#_flume_8217_s_dataflow_specification_language">8.2. Flume&#8217;s Dataflow Specification Language</a></span></dt><dd><dl><dt><span class="section"><a href="#_special_sinks_fan_out_fail_over_and_roll">8.2.1. Special Sinks: Fan out, Fail over, and Roll</a></span></dt><dt><span class="section"><a href="#_introducing_sink_decorators">8.2.2. Introducing Sink Decorators</a></span></dt><dt><span class="section"><a href="#_translations_of_high_level_sources_and_sinks">8.2.3. Translations of High-level Sources and Sinks</a></span></dt></dl></dd><dt><span class="section"><a href="#_custom_metadata_extraction">8.3. Custom Metadata Extraction</a></span></dt><dd><dl><dt><span class="section"><a href="#_extractors">8.3.1. Extractors</a></span></dt><dt><span class="section"><a href="#_meta_data_filtering_and_transformations">8.3.2. Meta Data Filtering and Transformations</a></span></dt><dt><span class="section"><a href="#_role_defaults">8.3.3. Role Defaults</a></span></dt><dt><span class="section"><a href="#_arbitrary_data_flows_and_custom_architectures">8.3.4. Arbitrary Data Flows and Custom Architectures</a></span></dt></dl></dd><dt><span class="section"><a href="#_extending_via_sink_source_decorator_plugins">8.4. Extending via Sink/Source/Decorator Plugins</a></span></dt><dd><dl><dt><span class="section"><a href="#_semantics_of_flume_extensions">8.4.1. Semantics of Flume Extensions</a></span></dt><dd><dl><dt><span class="section"><a href="#_simple_source_semantics">8.4.1.1. Simple source semantics.</a></span></dt><dt><span class="section"><a href="#_buffered_source_semantics">8.4.1.2. Buffered source semantics</a></span></dt><dt><span class="section"><a href="#_simple_sinks">8.4.1.3. Simple Sinks.</a></span></dt><dt><span class="section"><a href="#_buffered_sink_and_decorator_semantics">8.4.1.4. Buffered sink and decorator semantics</a></span></dt><dt><span class="section"><a href="#_retries_sleeps_and_unclean_exits">8.4.1.5. Retries, sleeps, and unclean exits.</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="#_limiting_data_transfer_rate_between_source_sink_pairs">8.5. Limiting Data Transfer Rate between Source-Sink pairs</a></span></dt></dl></dd><dt><span class="section"><a href="#_flume_and_hdfs_security_integration">9. Flume and HDFS Security Integration</a></span></dt><dd><dl><dt><span class="section"><a href="#_basics">9.1. Basics</a></span></dt><dt><span class="section"><a href="#_setting_up_flume_users_on_kerberos">9.2. Setting up Flume users on Kerberos</a></span></dt><dd><dl><dt><span class="section"><a href="#_administering_kerberos_principals">9.2.1. Administering Kerberos principals</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="#_appendix">10. Appendix</a></span></dt><dd><dl><dt><span class="section"><a href="#_flume_source_catalog">10.1. Flume Source Catalog</a></span></dt><dt><span class="section"><a href="#_flume_sinks_catalog">10.2. Flume Sinks Catalog</a></span></dt><dt><span class="section"><a href="#_flume_sink_decorator_catalog">10.3. Flume Sink Decorator Catalog</a></span></dt><dt><span class="section"><a href="#_flume_environment_variables">10.4. Flume Environment Variables</a></span></dt><dt><span class="section"><a href="#_flume_site_xml_configuration_settings">10.5. flume-site.xml configuration settings</a></span></dt><dt><span class="section"><a href="#_troubleshooting">10.6. Troubleshooting</a></span></dt><dd><dl><dt><span class="section"><a href="#_what_are_the_default_ports">10.6.1. What are the default ports?</a></span></dt><dt><span class="section"><a href="#_what_versions_of_hadoop_hdfs_can_i_use_how_do_i_change_this">10.6.2. What versions of Hadoop HDFS can I use?  How do I change this?</a></span></dt><dt><span class="section"><a href="#_why_doesn_8217_t_a_flume_node_appear_on_flume_master">10.6.3. Why doesn&#8217;t a Flume node appear on Flume Master?</a></span></dt><dt><span class="section"><a href="#_why_is_the_state_of_a_flume_node_changing_rapidly">10.6.4. Why is the state of a Flume node changing rapidly?</a></span></dt><dt><span class="section"><a href="#_where_are_the_logs_for_troubleshooting_flume_itself">10.6.5. Where are the logs for troubleshooting Flume itself?</a></span></dt><dt><span class="section"><a href="#_what_can_i_do_if_i_get_node_failure_due_to_out_of_file_handles">10.6.6. What can I do if I get node failure due to out of file handles?</a></span></dt><dt><span class="section"><a href="#_failures_due_when_using_disk_failover_or_write_ahead_log">10.6.7. Failures due when using Disk Failover or Write Ahead Log</a></span></dt><dt><span class="section"><a href="#_can_i_write_data_to_amazon_s3">10.6.8. Can I write data to Amazon S3?</a></span></dt></dl></dd></dl></dd><dt><span class="glossary"><a href="#_glossary">Glossary</a></span></dt><dt><span class="section"><a href="#_versions">11. Versions</a></span></dt><dd><dl><dt><span class="section"><a href="#_history">11.1. history</a></span></dt></dl></dd></dl></div><div class="list-of-figures"><p><b>List of Figures</b></p><dl><dt>1. <a href="#id415609">Flume Flows: Single Flow</a></dt><dt>2. <a href="#id415698">Flume Flows: Multiple Flows</a></dt><dt>3. <a href="#id416532">Flume Master: Standalone Mode</a></dt><dt>4. <a href="#id416555">Flume Master: Distributed Mode</a></dt></dl></div><div class="abstract" title="Abstract"><a name="_abstract"></a><p class="title"><b>Abstract</b></p><div class="abstract" title="Abstract"><p class="title"><b>Abstract</b></p><p>Flume is a distributed, reliable, and available service for
-efficiently collecting, aggregating, and moving large amounts of log
-data.  It has a simple and flexible architecture based on streaming
-data flows.  It is robust and fault tolerant with tunable reliability
-mechanisms and many failover and recovery mechanisms.  The system is
-centrally managed and allows for intelligent dynamic management. It
-uses a simple extensible data model that allows for online analytic
-applications.</p></div></div><div class="section" title="1.�Introduction"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="Introduction"></a>1.�Introduction</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_architecture">1.1. Architecture</a></span></dt><dt><span class="section"><a href="#_reliability">1.2. Reliability</a></span></dt><dt><span class="section"><a href="#_scalability">1.3. Scalability</a></span></dt><dt><span class="section"><a href="#_manageability">1.4. Manageability</a></span></dt><dt><span class="section"><a href="#_extensibility">1.5. Extensibility</a></span></dt><dt><span class="section"><a href="#_section_summary">1.6. Section summary</a></span></dt></dl></div><p>Flume is a distributed, reliable, and available service for
-efficiently moving large amounts of data soon after the data is
-produced.  This release provides a scalable conduit to move data
-around a cluster as well as reliable logging.</p><p>The primary use case for Flume is as a logging system that gathers a
-set of log files on every machine in a cluster and aggregates them to
-a centralized persistent store such as the Hadoop Distributed File
-System (HDFS).</p><p>The system was designed with these four key goals in mind:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Reliability
-</li><li class="listitem">
-Scalability
-</li><li class="listitem">
-Manageability
-</li><li class="listitem">
-Extensibility
-</li></ul></div><p>This section provides a high-level overview of Flume&#8217;s architecture
-and describes how the four design goals are achieved.</p><div class="section" title="1.1.�Architecture"><div class="titlepage"><div><div><h3 class="title"><a name="_architecture"></a>1.1.�Architecture</h3></div></div></div><p>Flume&#8217;s architecture is simple, robust, and flexible.  The main
-abstraction in Flume is a stream-oriented <span class="strong"><strong>data flow</strong></span>.  A data flow
-describes the way a single stream of data is transferred and processed
-from its point of generation to its eventual destination. Data flows
-are composed of <span class="strong"><strong>logical nodes</strong></span> that can transform or aggregate the
-events they receive.  Logical nodes are wired together in chains to
-form a data flow. The way in which they are wired is called the
-logical node&#8217;s <span class="strong"><strong>configuration</strong></span>.</p><p>Controlling all this is the <span class="strong"><strong>Flume Master</strong></span>, which is a separate
-service with knowledge of all the physical and logical nodes in a
-Flume installation. The Master assigns configurations to logical
-nodes, and is responsible for communicating configuration updates by
-the user to all logical nodes.  In turn, the logical nodes
-periodically contact the master so they can share monitoring
-information and check for updates to their configuration.</p><div class="informalfigure"><div class="mediaobject"><img src="architecture.png" alt="architecture.png"></div></div><p>The graph above shows a typical deployment of Flume that collects log
-data from a set of application servers.  The deployment consists of a
-number of <span class="emphasis"><em>logical nodes</em></span>, arranged into three tiers. The first tier
-is the <span class="emphasis"><em>agent</em></span> tier.  Agent nodes are typically installed on the
-machines that generate the logs and are your data&#8217;s initial point of
-contact with Flume.  They forward data to the next tier of <span class="emphasis"><em>collector
-nodes</em></span>, which aggregate the separate data flows and forward them to
-the final <span class="emphasis"><em>storage tier</em></span>.</p><p>For example, the agents could be machines listening for <code class="literal">syslog</code> data
-or monitoring the logs of a service such as a web server or the Hadoop
-JobTracker. The agents produce streams of data that are sent to the
-collectors; the collectors then aggregate the streams into larger
-streams which can be written efficiently to a storage tier such as
-HDFS.</p><p>Logical nodes are a very flexible abstraction. Every logical node has
-just two components - a <span class="emphasis"><em>source</em></span> and a <span class="emphasis"><em>sink</em></span>. The source tells a
-logical node where to collect data, and the sink tells it where to
-send the data. The only difference between two logical nodes is how
-the source and sink are configured. Both source and sink can
-additionally be configured with <span class="emphasis"><em>decorators</em></span> which perform some simple
-processing on data as it passes through. In the previous example, the
-collector and the agents are <span class="emphasis"><em>running the same node software</em></span>.  The
-Master assigns a configuration to each logical node at run-time - all
-components of a node&#8217;s configuration are instantiated dynamically at
-run-time, and therefore configurations can be changed many times
-throughout the lifetime of a Flume service without having to restart
-any Java processes or log into the machines themselves. In fact,
-logical nodes themselves can be created and deleted dynamically.</p><p>The source, sink, and optional decorators are a powerful set of
-primitives. Flume uses this architecture to provide per-flow data
-properties (for example durability guarantees, compression, or
-batching), or to compute event metadata, or even generate new events
-that are inserted into data flow.  A logical node can also send data
-downstream to several logical nodes.  This allows for multiple flows
-and each subflow can potentially be configured differently.  For
-example, it&#8217;s possible to have one flow be a collection path,
-delivering data reliably to a persistent store, while another branch
-computes lightweight analytics to be delivered to an alerting system.</p><div class="sidebar" title="Logical and Physical Nodes"><p class="title"><b>Logical and Physical Nodes</b></p><p>It&#8217;s important to make the distinction between <span class="emphasis"><em>logical nodes</em></span> and
-<span class="emphasis"><em>physical nodes</em></span>. A physical node corresponds to a single Java process
-running on one machine in a single JVM instance. Usually there is just
-one physical node per machine. Physical nodes act as containers for
-<span class="emphasis"><em>logical nodes</em></span>, which are wired together to form data flows. Each
-physical node can play host to many logical nodes, and takes care of
-arbitrating the assignment of machine resources between them. So,
-although the agents and the collectors in the preceding example are
-<span class="emphasis"><em>logically</em></span> separate processes, they could be running on the same
-<span class="emphasis"><em>physical</em></span> node. Flume gives users the flexibility to specify where
-the computation and data transfer are done. The remainder of this
-guide describes logical nodes unless stated otherwise.</p></div></div><div class="section" title="1.2.�Reliability"><div class="titlepage"><div><div><h3 class="title"><a name="_reliability"></a>1.2.�Reliability</h3></div></div></div><p>Reliability, the ability to continue delivering events in the face of
-failures without losing data, is a vital feature of Flume. Large
-distributed systems can and do suffer partial failures in many ways -
-physical hardware can fail, resources such as network bandwidth or
-memory can become scarce, or software can crash or run slowly. Flume
-emphasizes fault-tolerance as a core design principle and keeps
-running and collecting data even when many components have failed.</p><p>Flume can guarantee that all data received by an agent node will
-eventually make it to the collector at the end of its flow as long as
-the agent node keeps running.  That is, data can be <span class="strong"><strong>reliably</strong></span>
-delivered to its eventual destination.</p><p>However, reliable delivery can be very resource intensive and is often
-a stronger guarantee than some data sources require. Therefore, Flume
-allows the user to specify, on a per-flow basis, the level of
-reliability required. There are three supported reliability levels:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-End-to-end
-</li><li class="listitem">
-Store on failure
-</li><li class="listitem">
-Best effort
-</li></ul></div><div class="sidebar" title="A Note About Reliability"><p class="title"><b>A Note About Reliability</b></p><p>Although Flume is extremely tolerant to machine, network, and software
-failures, there is never any such thing as <span class="emphasis"><em>100% reliability</em></span>. If all
-the machines in a Flume installation were irrevocably destroyed in
-some terrible data center incident, all copies of Flume&#8217;s data would
-be lost and there would be no way to recover them. Therefore all of
-Flume&#8217;s reliability levels make guarantees about data delivery <span class="emphasis"><em>until
-some maximum number of failures have occurred</em></span>. Flume&#8217;s failure modes
-- in terms of what can fail and what will keep running if they do -
-are described in detail later in this guide.</p></div><p>The <span class="strong"><strong>end-to-end</strong></span> reliability level guarantees that once Flume accepts
-an event, that event will make it to the endpoint - as long as the
-agent that accepted the event remains live long enough. The first
-thing the agent does in this setting is write the event to disk in a
-<span class="emphasis"><em>'write-ahead log</em></span>' (WAL) so that, if the agent crashes and restarts,
-knowledge of the event is not lost. After the event has successfully
-made its way to the end of its flow, an acknowledgment is sent back to
-the originating agent so that it knows it no longer needs to store the
-event on disk. This reliability level can withstand any number of
-failures downstream of the initial agent.</p><p>The <span class="strong"><strong>store on failure</strong></span> reliability level causes nodes to only require
-an acknowledgement from the node one hop downstream.  If the sending
-node detects a failure, it will store data on its local disk until the
-downstream node is repaired, or an alternate downstream destination
-can be selected.  While this is effective, data can be lost if a
-compound or silent failure occurs.</p><p>The <span class="strong"><strong>best-effort</strong></span> reliability level sends data to the next hop with no
-attempts to confirm or retry delivery.  If nodes fail, any data that
-they were in the process of transmitting or receiving can be
-lost. This is the weakest reliability level, but also the most
-lightweight.</p><div class="sidebar" title="Eliminating Single Points of Failure"><p class="title"><b>Eliminating Single Points of Failure</b></p><p>Even when using the end-to-end reliability level, a Flume flow can
-fail to make progress if there aren&#8217;t any nodes available to process
-events. The data will continue to be collected by the agents, but the
-agents may not be able to deliver the data downstream until a suitable
-node comes online. In order to maintain high availability, Flume
-allows flows to fail over from one downstream node to another without
-any user intervention. The Flume Master can also be replicated for
-high availability which means that both the data plane and the control
-plane can tolerate a number of faults.</p></div></div><div class="section" title="1.3.�Scalability"><div class="titlepage"><div><div><h3 class="title"><a name="_scalability"></a>1.3.�Scalability</h3></div></div></div><p>Scalability is the ability to increase system performance linearly -
-or better - by adding more resources to the system.  Flume&#8217;s goal is
-horizontal scalability&#8201;&#8212;&#8201;the ability to incrementally add more
-machines to the system to increase throughput. A key performance
-measure in Flume is the number or size of events entering the system
-and being delivered.  When load increases, it is simple to add more
-resources to the system in the form of more machines to handle the
-increased load.</p><p>As seen in the preceding example installation, there are three
-separate components of Flume that require different approaches to
-scalability: the collector tier, the master, and the storage tier.</p><p>The collector tier needs to be able to scale in order to handle large
-volumes of data coming from large numbers of agent nodes.  This
-workload is write heavy, partitionable, and thus parallelizable.  By
-adding more machines to the collector tier, you can increase the
-number of agents and the maximum available throughput of the system.</p><p>An individual collector can usually handle many agents (up to
-hundreds) because each individual agent often produces only small
-amounts of log data compared to the full bandwidth available to the
-collector. Therefore, Flume balances flows from agents across
-different collectors. (One flow from an agent will talk to the same
-collector.) Flume uses a randomized algorithm to evenly assign lists
-of collectors to flows.  This automatically spreads the load, and also
-keeps the load spread in the case where a collector fails.</p><p>As the number of nodes in the system increases, the volume of traffic
-on the control path to and from the Flume Master may become a
-bottleneck. The Flume Master also supports horizontal scaling by
-adding more machines - although just a small number of commodity
-servers can serve a large installation of nodes. The state of the
-Flume Master is kept synchronized and fully replicated, which ensures
-that it is both fault tolerant and highly scalable.</p><p>Finally, Flume can only write data through a flow at the rate that the
-final destinations can accept. Although Flume is able to buffer data
-inside a flow to smooth out high-volume bursts, the output rate needs
-to be equal on average to the input rate to avoid log jams. Thus,
-writing to a scalable storage tier is advisable.  For example, HDFS
-has been shown to scale to thousands of machines and can handle many
-petabytes of data.</p></div><div class="section" title="1.4.�Manageability"><div class="titlepage"><div><div><h3 class="title"><a name="_manageability"></a>1.4.�Manageability</h3></div></div></div><p>Manageability is the ability to control data flows, monitor nodes,
-modify settings, and control outputs of a large system.  Manually
-managing the data flow from the sources to the end point is tedious,
-error prone, and a major pain point.  With the potential to have
-thousands of log-generating applications and services, it&#8217;s important
-to have a centralized management point to monitor and change data
-flows, and the ability to dynamically handle different conditions or
-problems.</p><p>The Flume Master is the point where global state such as the data
-flows can be managed. Via the Flume Master, users can monitor flows
-and reconfigure them on the fly.  The Flume Master has the information
-required to automatically respond to system changes such as load
-imbalances, partial failures, or newly provisioned hardware.</p><p>You can dynamically reconfigure nodes by using the Flume Master.
-Although this guide describes examples of traditional three-tier
-deployments, the flexibility of the nodes allow for arbitrary node
-topologies.  You can reconfigure nodes by using small scripts written
-in a flexible dataflow specification language, which can be submitted
-via the Flume Master interface.</p><p>You can administer the Flume Master by using either of two interfaces:
-a web interface or the scriptable Flume command shell.  The web
-interface provides interactive updates of the system&#8217;s state.  The
-shell enables administration via manually crafted scripts or
-machine-generated scripts.</p></div><div class="section" title="1.5.�Extensibility"><div class="titlepage"><div><div><h3 class="title"><a name="_extensibility"></a>1.5.�Extensibility</h3></div></div></div><p>Extensibility is the ability to add new functionality to a system. For
-example, you can extend Flume by adding connectors to existing storage
-layers or data platforms.  This is made possible by simple interfaces,
-separation of functional concerns into simple composable pieces, a
-flow specification language, and a simple but flexible data model.</p><p>Flume provides many common input and output connectors.  When new
-input connectors (sources) are added, extra metadata fields specific
-to that source can be attached to each event it produces.  Flume
-reuses the common components that provide particular reliability and
-resource usage properties.  Some general sources include files from
-the file system, syslog and syslog-ng emulation, or the standard
-output of a process.  More specific sources such as IRC channels and
-Twitter streams can also be added.  Similarly, there are many output
-destinations for events.  Although HDFS is the primary output
-destination, events can be sent to local files, or to monitoring and
-alerting applications such as Ganglia or communication channels such
-as IRC.</p><p>To enable easy integration with HDFS, MapReduce, and Hive, Flume
-provides simple mechanisms for output file management and output
-format management.  Data gathered by Flume can be processed easily
-with Hadoop and Hive.</p></div><div class="section" title="1.6.�Section summary"><div class="titlepage"><div><div><h3 class="title"><a name="_section_summary"></a>1.6.�Section summary</h3></div></div></div><p>The preceding Introduction section describes the high level goals and
-features of Flume. The following sections of this guide describe how
-to set up and use Flume:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Step-by-step tutorial that introduces a single Flume node
-</li><li class="listitem">
-Introduction to the Flume Master and a pseudo-distributed mode that
-  includes multiple nodes coordinated by the Flume Master
-</li><li class="listitem">
-Description of a fully-distributed setup that also removes single
-  points of failure
-</li><li class="listitem">
-Flume use cases and a description of how to integrate Flume with
-  existing sources of data
-</li><li class="listitem">
-How to set up Flume&#8217;s output so that integration with heavyweight
-  analysis systems such as Hadoop and Hive
-</li><li class="listitem">
-How to deploy Flume, set up arbitrary flows, and a specification of
-  Flume&#8217;s data flow specification language
-</li><li class="listitem">
-Catalog of components available via the language
-</li><li class="listitem">
-A description of experimental features
-</li><li class="listitem">
-Troubleshooting information
-</li></ul></div></div></div><div class="section" title="2.�Flume Single Node Quick Start"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="Quickstart"></a>2.�Flume Single Node Quick Start</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_sources_and_the_literal_dump_literal_command">2.1. Sources and the <code class="literal">dump</code> command</a></span></dt><dd><dl><dt><span class="section"><a href="#_reading_from_a_text_file_literal_text_literal">2.1.1. Reading from a text file, <code class="literal">text</code></a></span></dt><dt><span class="section"><a href="#_tailing_a_file_name_literal_tail_literal_and_literal_multitail_literal">2.1.2. Tailing a file name, <code class="literal">tail</code> and <code class="literal">multitail</code></a></span></dt><dt><span class="section"><a href="#_synthetic_sources_literal_synth_literal">2.1.3. Synthetic sources, <code class="literal">synth</code></a></span></dt><dt><span class="section"><a href="#_syslog_as_a_source_literal_syslogudp_literal_and_literal_syslogtcp_literal">2.1.4. Syslog as a source, <code class="literal">syslogUdp</code> and <code class="literal">syslogTcp</code></a></span></dt></dl></dd><dt><span class="section"><a href="#_anatomy_of_an_event">2.2. Anatomy of an Event</a></span></dt><dt><span class="section"><a href="#_section_summary_2">2.3. Section Summary</a></span></dt></dl></div><p>In this section, you will learn how to get a single Flume node running and
-transmitting data. You will also learn about some data <span class="strong"><strong>sources</strong></span>, and how to
-configure Flume flows on a per-node basis.</p><p>Each logical node consists of a event-producing <span class="strong"><strong>source</strong></span> and an event-
-consuming <span class="strong"><strong>sink</strong></span>.  Nodes pull data from their sources, and push data out
-through their sink.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>This section assumes that the Flume node and Flume Master are running in
-the foreground and not as daemons. You can stop the daemons by using <span class="emphasis"><em>/etc/
-init.d/flume-master stop</em></span> and <span class="emphasis"><em>/etc/init.d/flume-node stop</em></span>.</p></td></tr></table></div><div class="section" title="2.1.�Sources and the dump command"><div class="titlepage"><div><div><h3 class="title"><a name="_sources_and_the_literal_dump_literal_command"></a>2.1.�Sources and the <code class="literal">dump</code> command</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_reading_from_a_text_file_literal_text_literal">2.1.1. Reading from a text file, <code class="literal">text</code></a></span></dt><dt><span class="section"><a href="#_tailing_a_file_name_literal_tail_literal_and_literal_multitail_literal">2.1.2. Tailing a file name, <code class="literal">tail</code> and <code class="literal">multitail</code></a></span></dt><dt><span class="section"><a href="#_synthetic_sources_literal_synth_literal">2.1.3. Synthetic sources, <code class="literal">synth</code></a></span></dt><dt><span class="section"><a href="#_syslog_as_a_source_literal_syslogudp_literal_and_literal_syslogtcp_literal">2.1.4. Syslog as a source, <code class="literal">syslogUdp</code> and <code class="literal">syslogTcp</code></a></span></dt></dl></div><p>Start by getting a Flume node running that echoes data written to standard
-input from the console back out to the console on stdout. You do this by using
-the <code class="literal">dump</code> command.</p><pre class="screen">$ flume dump console</pre><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>The Flume program has the general form <code class="literal">flume &lt;command&gt; [args ...]</code>.  If
-you installed from the tarball package, the command can be found in
-<code class="literal">$FLUME_HOME/bin/</code>. If you installed from either RPM or DEB, then <code class="literal">flume</code>
-should already be in your path.</p></td></tr></table></div><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>The example above uses the <code class="literal">dump</code> command and <code class="literal">console</code> is the argument.
-The command&#8217;s syntax is <code class="literal">flume dump &lt;source&gt;</code>. It prints data from <code class="literal">&lt;source&gt;</code>
-to the console.</p></td></tr></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Some flume configurations by default write to local disk.
-Initially the default is <span class="emphasis"><em>/tmp/flume</em></span>.  This is good for initial
-testing but for production environments the <code class="literal">flume.agent.logdir</code>
-property should be set to a more durable location.</p></td></tr></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>If the node refuses to run and exits with this message,
-<code class="literal">agent.FlumeNode: Aborting: Unexpected problem with
-environment.Failure to write in log directory: <span class="emphasis"><em>/tmp/flume</em></span>.  Check
-permissions?</code>, then check the <code class="literal">/ tmp/flume</code> directory to make sure you
-have write permissions to it (change the owner or have the user join
-the group).  This is, by default, where various logging information is
-kept.</p></td></tr></table></div><p>You have started a Flume node where <code class="literal">console</code> is the source of incoming data.
-When you run it, you should see some logging messages displayed to the
-console.  For now, you can ignore messages about Masters, back-off and failed
-connections (these are explained in later sections).  When you type at the
-console and press a new line, you should see a new log entry line appear
-showing the data that you typed. If you entered <code class="literal">This is a test</code>, it should
-look similar to this:</p><pre class="screen">hostname [INFO Thu Nov 19 08:37:13 PST 2009] This is a test</pre><p>To exit the program, press ^C.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Some sources do not automatically exit and require a manual ^C to
-exit.</p></td></tr></table></div><div class="section" title="2.1.1.�Reading from a text file, text"><div class="titlepage"><div><div><h4 class="title"><a name="_reading_from_a_text_file_literal_text_literal"></a>2.1.1.�Reading from a text file, <code class="literal">text</code></h4></div></div></div><p>You can also specify other sources of events.  For example, if you want a text
-file where each line represents a new event, run the following command.</p><pre class="screen">$ flume dump 'text("/etc/services")'</pre><p>This command reads the file, and then outputs each line as a new event.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>The default console output escapes special characters with Java-style
-escape sequences. Characters such as <span class="emphasis"><em>"</em></span> and <span class="emphasis"><em>\</em></span> are prefaced with an extra
-<span class="emphasis"><em>\</em></span>.</p></td></tr></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>You can try this command with other files such as <code class="literal">/var/log/messages</code>,
-<code class="literal">/var/log/syslog</code>, or <code class="literal">/var/log/hadoop/hadoop.log</code> also.  However, Flume must
-run with appropriate permissions to read the files.</p></td></tr></table></div></div><div class="section" title="2.1.2.�Tailing a file name, tail and multitail"><div class="titlepage"><div><div><h4 class="title"><a name="_tailing_a_file_name_literal_tail_literal_and_literal_multitail_literal"></a>2.1.2.�Tailing a file name, <code class="literal">tail</code> and <code class="literal">multitail</code></h4></div></div></div><p>If you want to tail a file instead of just reading it, specify another source
-by using <code class="literal">tail</code> instead of <code class="literal">text</code>.</p><pre class="screen">$ flume dump 'tail("testfile")'</pre><p>This command pipes data from the file into Flume and then out to the console.</p><p>This message appears: "File <span class="emphasis"><em>testfile</em></span> does not currently exist, waiting for
-file to appear".</p><p>In another terminal, you can create and write data to the file:</p><pre class="screen">$ echo Hello world! &gt;&gt; testfile</pre><p>New data should appear.</p><p>When you delete the file:</p><pre class="screen">$ rm testfile</pre><p>The <code class="literal">tail</code> sink detects this.  If you then recreate the file, the <code class="literal">tail</code>
-source detects the new file and follows it:</p><pre class="screen">$ echo Hello world again! &gt;&gt; testfile</pre><p>You should see your new message appear in the Flume node console.</p><p>You can also use the <code class="literal">multitail</code> source to follow multiple files by file name:</p><pre class="screen">$ flume dump 'multitail("test1", "test2")'</pre><p>And send it data coming from the two different files:</p><pre class="screen">$ echo Hello world test1! &gt;&gt; test1
-$ echo Hello world test2! &gt;&gt; test2</pre></div><div class="section" title="2.1.3.�Synthetic sources, synth"><div class="titlepage"><div><div><h4 class="title"><a name="_synthetic_sources_literal_synth_literal"></a>2.1.3.�Synthetic sources, <code class="literal">synth</code></h4></div></div></div><p>Here&#8217;s one more example where you use the <code class="literal">synth</code> sources to generate events:</p><pre class="screen">$ flume dump 'asciisynth(20,30)'</pre><p>You should get 20 events, each with 30 random ASCII bytes.</p></div><div class="section" title="2.1.4.�Syslog as a source, syslogUdp and syslogTcp"><div class="titlepage"><div><div><h4 class="title"><a name="_syslog_as_a_source_literal_syslogudp_literal_and_literal_syslogtcp_literal"></a>2.1.4.�Syslog as a source, <code class="literal">syslogUdp</code> and <code class="literal">syslogTcp</code></h4></div></div></div><p>As with files, you can also accept data from well known wire formats such as
-syslog. For example, you can start a traditional syslog-like UDP server
-listening on port 5140 (the normal syslog UDP port is the privileged port 514)
-by running this command:</p><pre class="screen">$ flume dump 'syslogUdp(5140)'</pre><p>You can feed the source data by using netcat to send syslog formatted data as
-shown in the example below:</p><pre class="screen">$ echo "<a name="CO1-1"></a><img src="images/callouts/1.png" alt="1" border="0">hello via syslog"  | nc -u localhost 5140</pre><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>You may need to press ^C to exit this command.</p></td></tr></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>The extra <code class="literal">&lt;37&gt;</code> is a syslog wireformat encoding of a message category
-and priority level.</p></td></tr></table></div><p>Similarly, you can set up a syslog-ng compatible source that listens on TCP
-port 5140 (the normal syslog-ng TCP port is the privileged port 514):</p><pre class="screen">$ flume dump 'syslogTcp(5140)'</pre><p>And send it data:</p><pre class="screen">$ echo "<a name="CO1-2"></a><img src="images/callouts/1.png" alt="1" border="0">hello via syslog" | nc -t localhost 5140</pre><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>You may need to press ^C to exit this command.</p></td></tr></table></div><p>Syslog backwards-compatibility allows data normally created from syslog,
-rsyslog, or syslog-ng to be sent to and processed by Flume.</p></div></div><div class="section" title="2.2.�Anatomy of an Event"><div class="titlepage"><div><div><h3 class="title"><a name="_anatomy_of_an_event"></a>2.2.�Anatomy of an Event</h3></div></div></div><p>This section describes a number of sources of data that Flume can interoperate
-with. Before going any further, it will be helpful for you to understand what
-Flume is actually sending and processing internally.</p><p>Flume internally converts every external source of data into a stream of
-<span class="strong"><strong>events</strong></span>. Events are Flume&#8217;s unit of data and are a simple and flexible
-representation. An event is composed of a <span class="strong"><strong>body</strong></span> and <span class="strong"><strong>metadata</strong></span>. The event
-body is a string of bytes representing the content of an event. For example, a
-line in a log file is represented as an event whose body was the actual byte
-representation of that line. The event metadata is a table of key / value
-pairs that capture some detail about the event, such as the time it was
-created or the name of the machine on which it originated. This table can be
-appended as an event travels along a Flume flow, and the table can be read to
-control the operation of individual components of that flow. For example, the
-machine name attached to an event can be used to control the output path where
-the event is written at the end of the flow.</p><p>An event&#8217;s body can be up to 32KB long - although this limit can be controlled
-via a system property, it is recommended that it is not changed in order to
-preserve performance.</p></div><div class="section" title="2.3.�Section Summary"><div class="titlepage"><div><div><h3 class="title"><a name="_section_summary_2"></a>2.3.�Section Summary</h3></div></div></div><p>In this section, you learned how to use Flume&#8217;s <code class="literal">dump</code> command to print data
-from a variety of different input sources to the console. You also learned
-about the <span class="strong"><strong>event</strong></span>, the fundamental unit of data transfer in Flume.</p><p>The following table summarizes the sources described in this section.</p><div class="variablelist" title="Flume Event"><p class="title"><b>Flume Event</b></p><dl><dt><span class="term">
-Sources <code class="literal">console</code> 
-</span></dt><dd>
-Stdin console
-</dd><dt><span class="term">
-<code class="literal">text("filename")</code> 
-</span></dt><dd>
-One shot text file source.  One line is one event
-</dd><dt><span class="term">
-<code class="literal">tail("filename")</code> 
-</span></dt><dd>
-Similar to Unix&#8217;s <code class="literal">tail -F</code>. One line is one event.
-Stays open for more data and follows filename if file rotated.
-</dd><dt><span class="term">
-<code class="literal">multitail("file1"[, "file2"[, &#8230;]])</code> 
-</span></dt><dd>
-Similar to <code class="literal">tail</code> source but follows
-multiple files.
-</dd><dt><span class="term">
-<code class="literal">asciisynth(msg_count,msg_size)</code> 
-</span></dt><dd>
-A source that synthetically generates
-msg_count random messages of size msg_size.  This converts all characters into
-printable ASCII characters.
-</dd><dt><span class="term">
-<code class="literal">syslogUdp(port)</code> 
-</span></dt><dd>
-Syslog over UDP port, port.  This is syslog compatible.
-</dd><dt><span class="term">
-<code class="literal">syslogTcp(port)</code> 
-</span></dt><dd>
-Syslog over TCP port, port. This is syslog-ng compatible.
-</dd></dl></div></div></div><div class="section" title="3.�Pseudo-distributed Mode"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_pseudo_distributed_mode"></a>3.�Pseudo-distributed Mode</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_starting_pseudo_distributed_flume_daemons">3.1. Starting Pseudo-distributed Flume Daemons</a></span></dt><dd><dl><dt><span class="section"><a href="#_the_master">3.1.1. The Master</a></span></dt><dt><span class="section"><a href="#_the_flume_node">3.1.2. The Flume Node</a></span></dt></dl></dd><dt><span class="section"><a href="#_configuring_a_node_via_the_master">3.2. Configuring a Node via the Master</a></span></dt><dt><span class="section"><a href="#_introducing_sinks">3.3. Introducing Sinks</a></span></dt><dt><span class="section"><a href="#_aggregated_configurations">3.4. Aggregated Configurations</a></span></dt><dt><span class="section"><a href="#_tiering_flume_nodes_agents_and_collectors">3.5. Tiering Flume Nodes: Agents and Collectors</a></span></dt><dt><span class="section"><a href="#_section_summary_3">3.6. Section Summary</a></span></dt></dl></div><p>Flume is intended to be run as a distributed system with processes spread out
-across <span class="emphasis"><em>many</em></span> machines.  It can also be run as several processes on a <span class="emphasis"><em>single</em></span>
-machine, which is called &#8220;pseudo-distributed&#8221; mode.  This mode is useful for
-debugging Flume data flows and getting a better idea of how Flume components
-interact.</p><p>The previous section described a Flume node and introduced the concept of
-Flume sources.  This section introduces some new concepts required for a
-distributed setup: the Flume <span class="strong"><strong>master</strong></span> server, the specification of sources and
-<span class="strong"><strong>sinks</strong></span>, and connecting multiple Flume <span class="strong"><strong>nodes</strong></span>.</p><div class="section" title="3.1.�Starting Pseudo-distributed Flume Daemons"><div class="titlepage"><div><div><h3 class="title"><a name="_starting_pseudo_distributed_flume_daemons"></a>3.1.�Starting Pseudo-distributed Flume Daemons</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_the_master">3.1.1. The Master</a></span></dt><dt><span class="section"><a href="#_the_flume_node">3.1.2. The Flume Node</a></span></dt></dl></div><p>There are two kinds of processes in the system: the Flume <span class="strong"><strong>master</strong></span> and the
-Flume <span class="strong"><strong>node</strong></span>.  The Flume Master is the central management point and controls
-the data flows of the nodes.  It is the single logical entity that holds
-global state data and controls the Flume node data flows and monitors Flume
-nodes.  Flume nodes serve as the data path for streams of events.  They can be
-the sources, conduits, and consumers of event data.  The nodes periodically
-contact the Master to transmit a heartbeat and to get their data flow
-configuration.</p><p>In order to get a distributed Flume system working, you must start a single
-Flume Master and some Flume nodes that interact with the Master. You&#8217;ll start
-with a Master and one Flume node and then expand.</p><div class="section" title="3.1.1.�The Master"><div class="titlepage"><div><div><h4 class="title"><a name="_the_master"></a>3.1.1.�The Master</h4></div></div></div><p>The Master can be manually started by executing the following command:</p><pre class="screen">$ flume master</pre><p>After the Master is started, you can access it by pointing a web browser to
-<a class="ulink" href="http://localhost:35871/" target="_top">http://localhost:35871/</a>.  This web page displays the status of all Flume nodes
-that have contacted the Master, and shows each node&#8217;s currently assigned
-configuration.  When you start this up without Flume nodes running, the status
-and configuration tables will be empty.</p><div class="informalfigure"><div class="mediaobject"><img src="master-empty.png" alt="master-empty.png"></div></div><p>The web page contains four tables&#8201;&#8212;&#8201;the <span class="emphasis"><em>Node status</em></span> table, the <span class="emphasis"><em>Node
-configuration</em></span> table, the <span class="emphasis"><em>Physical/Logical Node mapping</em></span> table, and a
-<span class="emphasis"><em>Command history</em></span> table.  The information in these tables represent the
-current global state of the Flume system.</p><p>The Master&#8217;s <span class="emphasis"><em>Node status</em></span> table contains the names of all of the Flume Nodes
-talking to the Master, their current configuration version (initially "none"),
-their status (such as IDLE), and when it last reported to the Master.  The
-name of each Flume node should be the same as running <code class="literal">hostname</code> from Unix
-prompt.</p><p>The Master&#8217;s <span class="emphasis"><em>Node configuration</em></span> table contains the logical name of a node,
-the configuration version assigned to it, and a specification of its sources
-and its sinks.  Initially, this table is empty, but after you change values
-you can view this web page to see the updates. There are two sets of columns -
-- the user entered version/source/sink, and translated version/source/sink. A
-later section of this guide describes translated configs.</p><p>The Master&#8217;s <span class="emphasis"><em>Physical/Logical Node mapping</em></span> table contains the mapping of
-logical nodes to their physical nodes.</p><p>The Master&#8217;s <span class="emphasis"><em>Command history</em></span> table contains the state of commands. In
-general, <span class="strong"><strong>commands</strong></span> modify the Master&#8217;s global state.  Commands are processed
-serially on a Master and are assigned a unique command ID number. Each command
-has a state (for example, SUCCEEDED, FAILED, or PENDING), a command line, and
-a message which often contains information about its execution attempt.</p></div><div class="section" title="3.1.2.�The Flume Node"><div class="titlepage"><div><div><h4 class="title"><a name="_the_flume_node"></a>3.1.2.�The Flume Node</h4></div></div></div><p>To start a Flume node, invoke the following command in another terminal.</p><pre class="screen">$ flume node_nowatch</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Normally, you start a node using <code class="literal">flume node</code> or run the node as a
-daemon.  For these examples, you disable the watchdog feature by starting with
-the <code class="literal">node_nowatch</code> option.  This option enables you to interact with a node
-via the console. The other options disable <code class="literal">stdin</code>.</p></td></tr></table></div><p>To check whether a Flume node is up, point your browser to the <span class="emphasis"><em>Flume Node
-status page</em></span> at <a class="ulink" href="http://localhost:35862/" target="_top">http://localhost:35862/</a>.  Each node displays its own data on a
-single table that includes diagnostics and metrics data about the node, its
-data flows, and the system metrics about the machine it is running on.</p><p>If the node is up, you should also refresh the Master&#8217;s status page (http://
-localhost:35871) to make sure that the node has contacted the Master. You
-brought up one node (assume the node is named <code class="literal"><span class="emphasis"><em>host</em></span></code>), so you should have
-one node listed in the Master&#8217;s node status table, and an entry in the logical
-node mapping table that links the <code class="literal"><span class="emphasis"><em>host</em></span></code> logical node to the <code class="literal"><span class="emphasis"><em>host</em></span></code>
-physical nodes.</p></div></div><div class="section" title="3.2.�Configuring a Node via the Master"><div class="titlepage"><div><div><h3 class="title"><a name="_configuring_a_node_via_the_master"></a>3.2.�Configuring a Node via the Master</h3></div></div></div><p>Requiring nodes to contact the Master to get their configuration enables you
-to dynamically change the configuration of nodes without having to log in to
-the remote machine to restart the daemon.  You can quickly change the node&#8217;s
-previous data flow configuration to a new one.</p><p>The following describes how to "wire" nodes using the Master&#8217;s web interface.</p><p>On the Master&#8217;s web page, click on the config link. You are presented with two
-forms.  These are web interfaces for setting the node&#8217;s data flows.  When
-Flume nodes contact the Master, they will notice that the data flow version
-has changed, instantiate, and activate the configuration.</p><p>For this example, you will do the steps you did in the Quick Start section.
-Enter the following values into the "Configure a node" form, and then click
-Submit.</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; "><colgroup><col align="left"><col align="left"></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Node name:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal"><span class="emphasis"><em>host</em></span></code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Source:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal">console</code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; " align="left">
-    Sink:
-    </td><td style="" align="left">
-    <code class="literal">console</code>
-    </td></tr></tbody></table></div><p>Refresh the Master page and notice that the version stamp changed to a current
-time, and that the src and sink fields of the configs updated.  After the
-status changes to "ACTIVE", it is ready to receive console traffic.</p><p>On the master, a node can be in one of several states:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-HELLO : A new node instance initially contacted the master.
-</li><li class="listitem">
-IDLE : A node has completed its configuration or has no configuration.
-</li><li class="listitem">
-CONFIGURING: A node has received a configuration and is activating the configuration.
-</li><li class="listitem">
-ACTIVE: A node is actively pulling data from the source and pushing data into the sink.
-</li><li class="listitem">
-LOST: A node has not contacted the master for an extended period of time (default is after 10x the expected heartbeat period&#8201;&#8212;&#8201;50s by default)
-</li><li class="listitem">
-DECOMMISSIONED: A node has been purposely decommissioned from a master.
-</li><li class="listitem">
-ERROR: A node has stopped in an error state.
-</li></ul></div><p>On the terminal where your Flume node is running, you should be able to type a
-few lines and then get output back showing your new log message.</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; "><colgroup><col align="left"><col align="left"></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Node name:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal"><span class="emphasis"><em>host</em></span></code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Source:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal">text("/etc/services")</code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; " align="left">
-    Sink:
-    </td><td style="" align="left">
-    <code class="literal">console</code>
-    </td></tr></tbody></table></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>You may need to press Enter in the Flume node console.</p></td></tr></table></div><p>Or use these new values to tail a file:</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; "><colgroup><col align="left"><col align="left"></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Node name:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal"><span class="emphasis"><em>host</em></span></code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Source:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal">tail("/etc/services")</code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; " align="left">
-    Sink:
-    </td><td style="" align="left">
-    <code class="literal">console</code>
-    </td></tr></tbody></table></div><p>You can now change the configuration of different nodes in the system to
-gather data from a variety of sources by going through the Master.</p></div><div class="section" title="3.3.�Introducing Sinks"><div class="titlepage"><div><div><h3 class="title"><a name="_introducing_sinks"></a>3.3.�Introducing Sinks</h3></div></div></div><p>Thus far, you have seen that Flume has a variety of sources that generate or
-accept new events that are fed into the system.  You have limited the output
-of these messages to the <code class="literal">console</code> sink.  As you would expect, Flume also
-provides a wide variety of event <span class="strong"><strong>sinks</strong></span>&#8201;&#8212;&#8201;destinations for all of the
-events.</p><p>There are many possible destinations for events&#8201;&#8212;&#8201;to local disk, to HDFS, to
-the console, or forwarding across the network. You use the sink abstractions
-as an interface for forwarding events to any of these destinations.</p><p>You can connect different sources to different sinks by specifying the new
-configuration and submitting it to the Master.  For example, with the data
-flow below, you can make a copy of <code class="literal">/etc/services</code>.</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; "><colgroup><col align="left"><col align="left"></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Node name:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal"><span class="emphasis"><em>host</em></span></code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Source:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal">text("/etc/services")</code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; " align="left">
-    Sink:
-    </td><td style="" align="left">
-    <code class="literal">text("services.copy")</code>
-    </td></tr></tbody></table></div><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>The <code class="literal">text</code> sinks overwrite if a file previously exists.</p></td></tr></table></div><p>Notice that the file is copied as is. Sinks have optional arguments for output
-format, which you can use to write data in other serialization formats.  For
-example, instead of copying a file using the default "raw" formatter, you can
-format the output using other formatters such as "avrojson" (the Avro
-serialization json format), "avrodata" (the Avro serialization binary data
-format), or a "debug" mode (this is the formatter used by the console sink).</p><p>If you enter:</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; "><colgroup><col align="left"><col align="left"></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Node name:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal"><span class="emphasis"><em>host</em></span></code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Source:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal">text("/etc/services")</code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; " align="left">
-    Sink:
-    </td><td style="" align="left">
-    <code class="literal">console("avrojson")</code>
-    </td></tr></tbody></table></div><p>You get the file with each record in JSON format displayed to the console.</p><p>If you enter:</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; "><colgroup><col align="left"><col align="left"></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Node name:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal"><span class="emphasis"><em>host</em></span></code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left">
-    Source:
-    </td><td style="border-bottom: 0.5pt solid ; " align="left">
-    <code class="literal">text("/etc/services")</code>
-    </td></tr><tr><td style="border-right: 0.5pt solid ; " align="left">
-    Sink:
-    </td><td style="" align="left">
-    <code class="literal">text("services.json", "avrojson")</code>
-    </td></tr></tbody></table></div><p>The newly written local <code class="literal">services.json</code> file is output in avro&#8217;s json format.</p><div class="sidebar" title="Default Arguments"><p class="title"><b>Default Arguments</b></p><p>Many sinks, sources, and decorators have arguments that modify their behavior.
-Some of these arguments are required; others are optional and use a default.</p><p>The following notation describes these parameters:</p><p><code class="literal"><span class="emphasis"><em>name</em></span> (<span class="emphasis"><em>req1</em></span>, <span class="emphasis"><em>req2</em></span>[, <span class="emphasis"><em>opt1</em></span>[, <span class="emphasis"><em>opt2</em></span>=<span class="emphasis"><em>foo</em></span>]] )</code></p><p>where a component with name <span class="emphasis"><em>name</em></span>, has two required arguments, <span class="emphasis"><em>req1</em></span> and
-<span class="emphasis"><em>req2</em></span>. There are two optional arguments, <span class="emphasis"><em>opt1</em></span> and <span class="emphasis"><em>opt2</em></span>.  However, <span class="emphasis"><em>opt1</em></span>
-<span class="strong"><strong>must</strong></span> be specified if <span class="emphasis"><em>opt2</em></span> is specified.  <span class="emphasis"><em>foo</em></span> in this case is the default
-argument for <span class="emphasis"><em>opt2</em></span>+.</p><p>For the <code class="literal">console</code> examples above, console could be specified as:</p><p><code class="literal">console[(<span class="emphasis"><em>format</em></span>="debug")]</code></p><p>and text could be specified as:</p><p><code class="literal">text("<span class="emphasis"><em>file</em></span>"[, <span class="emphasis"><em>format</em></span>="raw"])</code></p></div><p>There are several sinks you can use. The following list is a subset; see the
-Appendix for more sinks.</p><div class="horizontal"><a name="id413903"></a><p class="title"><b>Table�1.�Flume Event Sinks</b></p><div class="horizontal-contents"><table summary="Flume Event Sinks" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">null</code>  
-</p>
-</td><td style="" valign="top">
-<p>
-Null sink. Events are dropped.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">console[("format")]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Console sink.  Display to console&#8217;s stdout. The
-"format" argument is optional and defaults to the "debug" output format.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">text("<span class="emphasis"><em>txtfile</em></span>"[,"format"])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Textfile sink.  Write the events to text
-file <code class="literal"><span class="emphasis"><em>txtfile</em></span></code> using output format "format".  The default format is "raw"
-event bodies with no metadata.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">dfs("<span class="emphasis"><em>dfsfile</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-DFS seqfile sink.  Write serialized Flume events
-to a dfs path such as <code class="literal">hdfs://namenode/file</code> or <code class="literal">file:///file</code> in
-Hadoop&#8217;s seqfile format.  Note that because of the HDFS write
-semantics, no data for this sink write until the sink is closed.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">syslogTcp("<span class="emphasis"><em>host</em></span>",<span class="emphasis"><em>port</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Syslog TCP sink.  Forward to events to <code class="literal">host</code>
-on TCP port <code class="literal">port</code> in syslog wire format (syslog-ng compatible), or to other
-Flume nodes setup to listen for syslogTcp.
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>Using <code class="literal">dfs</code> has some restrictions and requires some extra
-setup.  Files contents will not become available until after the sink
-has been closed.  See the Troubleshooting section for details.</p></td></tr></table></div></div><div class="section" title="3.4.�Aggregated Configurations"><div class="titlepage"><div><div><h3 class="title"><a name="_aggregated_configurations"></a>3.4.�Aggregated Configurations</h3></div></div></div><p>Using the form for single node configuration of a small number of
-machines is manageable, but for larger numbers it is more efficient to
-maintain or auto-generate the configuration for all of the machines in
-a single file. Flume allows you to set the configurations of many
-machines in a single aggregated configuration submission.</p><p>Instead of the method you used in the "Configuring a Node via the Master"
-section, put the following configuration line into the "Configure Nodes" form
-and then submit:</p><pre class="screen">host : text("/etc/services") | console ;</pre><p>Or:</p><pre class="screen">host: text("/etc/services") | text("services.copy");</pre><p>The general format is:</p><pre class="screen">&lt;node1&gt; : &lt;source&gt; | &lt;sink&gt; ;
-&lt;node2&gt; : &lt;source&gt; | &lt;sink&gt; ;
-&lt;node3&gt; : &lt;source&gt; | &lt;sink&gt; ;
-...</pre><p>The remainder of this guide uses this format to configure nodes.</p></div><div class="section" title="3.5.�Tiering Flume Nodes: Agents and Collectors"><div class="titlepage"><div><div><h3 class="title"><a name="_tiering_flume_nodes_agents_and_collectors"></a>3.5.�Tiering Flume Nodes: Agents and Collectors</h3></div></div></div><p>A simple network connection is abstractly just another sink.  It would be
-great if sending events over the network was easy, efficient, and reliable.
-In reality, collecting data from a distributed set of machines and relying on
-networking connectivity greatly increases the likelihood and kinds of failures
-that can occur.  The bottom line is that providing reliability guarantees
-introduces complexity and many tradeoffs.</p><p>Flume simplifies these problems by providing a predefined topology and tunable
-reliability that only requires you to give each Flume node a role.  One simple
-Flume node topology classifies Flume nodes into two roles&#8201;&#8212;&#8201;a Flume <span class="strong"><strong>agent</strong></span>
-tier and the Flume <span class="strong"><strong>collector</strong></span> tier. The agent tier Flume nodes are co-located
-on machines with the service that is producing logs.  For example, you could
-specify a Flume agent configured to have <code class="literal">syslogTcp</code> as a source, and
-configure the syslog generating server to send its logs to the specified local
-port.  This Flume agent would have an <code class="literal">agentSink</code> as its sink which is
-configured to forward data to a node in the collector tier.</p><p>Nodes in the collector tier listen for data from multiple agents, aggregates
-logs, and then eventually write the data to HDFS.</p><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>In the next few sections, all <code class="literal"><span class="emphasis"><em>host</em></span></code> arguments currently use the
-physical IP or DNS name of the node as opposed to the Flume node name (set by
-<code class="literal">-n name</code> option). The default Flume node name is the host name unless you
-override it on the command line.</p></td></tr></table></div><p>To demonstrate the new sinks in pseudo-distributed mode, you will instantiate
-another Flume node (a physical node) on the local box.  To do this, you need
-to start a Flume node with some extra options.  The command line below starts
-a physical node named <code class="literal">collector</code> (<code class="literal">-n collector</code>):</p><pre class="screen">$ flume node_nowatch -n collector</pre><p>On the Master&#8217;s web page, you should eventually see two nodes: <code class="literal"><span class="emphasis"><em>host</em></span></code> and
-<code class="literal">collector</code>.  The Flume Node status web pages should be available at http://
-localhost:35862 and <a class="ulink" href="http://localhost:35863" target="_top">http://localhost:35863</a>.  Port bindings are dependent on
-instantiation order&#8201;&#8212;&#8201;the first physical node instantiated binds on 35862 and
-the second binds to 35863.</p><p>Next, configure <code class="literal">collector</code> to take on the role of a collector, set up
-<code class="literal"><span class="emphasis"><em>host</em></span></code> to send data from the console to the collector by using the
-aggregated multiple configuration form.  The agent uses the <code class="literal">agentSink</code>, a
-high reliability network sink.  The collector node&#8217;s source is configured to
-be a <code class="literal">collectorSource</code>, and its sink is configured to be the <code class="literal">console</code>.</p><pre class="screen">host : console | agentSink("localhost",35853) ;
-collector : collectorSource(35853) | console ;</pre><div class="informalfigure"><div class="mediaobject"><img src="tiers-console.png" alt="tiers-console.png"></div></div><p>When you type lines in <code class="literal"><span class="emphasis"><em>host</em></span></code>'s console, events are forwarded to the
-collector.  Currently, there is a bit of latency (15s or so) before the
-forwarded message shows up on <code class="literal">collector</code>.  This is actually a configurable
-setting whose default is set to a value that is amenable for high event
-throughputs.  Later sections describe how to tune Flume.</p><p>You have successfully made a event flow from an agent to the collector.</p><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>You can check to see if <code class="literal">agentSink</code> is working by looking in the
-write- ahead logging directory for the sink.  The default location is
-<code class="literal">/tmp/flume/ &lt;nodename&gt;</code>.  Because the OS periodically cleans this
-directory, this configuration property (<code class="literal">flume.agent.logdir</code>) should
-be changed a production environment.</p></td></tr></table></div><p>A more interesting setup is to have the agent tailing a local file (using the
-<code class="literal">tail</code> source) or listening for local syslog data (using the <code class="literal">syslogTcp</code> or
-<code class="literal">syslogUdp</code> sources and modifying the syslog daemon&#8217;s configuration).  Instead
-of writing to a console, the collector would write to a <code class="literal">collectorSink</code>, a
-smarter sink that writes to disk or HDFS, periodically rotates files, and
-manages acknowledgements.</p><div class="informalfigure"><div class="mediaobject"><img src="tiers-hdfs.png" alt="tiers-hdfs.png"></div></div><p>The following configuration is for an agent that listens for syslog messages
-and forwards to a collector which writes files to local directory <code class="literal">/tmp/flume/
-collected/</code>.</p><pre class="screen">host : syslogTcp(5140) | agentSink("localhost",35853) ;
-collector : collectorSource(35853) | collectorSink("file:///tmp/flume/collected", "syslog");</pre><p>In the following slightly modified configuration, the collector writes to an
-HDFS cluster (assuming the HDFS nameNode is called <code class="literal">namenode</code>):</p><pre class="screen">host : syslogTcp(5140) | agentSink("localhost",35853) ;
-collector : collectorSource(35853) | collectorSink("hdfs://namenode/user/flume/
-","syslog");</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>There are no guarantees that data written to an HDFS file is
-durable until the HDFS file is properly closed.  Because of this, the
-collector sink periodically closes a file and creates a new one in
-HDFS.  The default time between file rolls (close then open new) is
-30s.  If you are writing data at low throughput (&lt;2MB/s) you may want
-to increase the default time by modifying the
-<code class="literal">flume.collector.roll.millis</code> and <code class="literal">flume.agent.logdir.retransmit</code> time
-properties in your flume-site.xml file.</p></td></tr></table></div></div><div class="section" title="3.6.�Section Summary"><div class="titlepage"><div><div><h3 class="title"><a name="_section_summary_3"></a>3.6.�Section Summary</h3></div></div></div><p>This section describes how to start a Master and a node, and configure a node
-via the Master. The next section describes a more concise way of specifying
-many configurations, explains agents and collectors, and how to build an
-agent-collector pipeline on a single machine in a three-tier topology.</p></div></div><div class="section" title="4.�Fully-distributed Mode"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_fully_distributed_mode"></a>4.�Fully-distributed Mode</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_static_configuration_files">4.1. Static Configuration Files</a></span></dt><dd><dl><dt><span class="section"><a href="#_using_default_values">4.1.1. Using Default Values</a></span></dt></dl></dd><dt><span class="section"><a href="#_multiple_collectors">4.2. Multiple Collectors</a></span></dt><dd><dl><dt><span class="section"><a href="#_partitioning_agents_across_multiple_collectors">4.2.1. Partitioning Agents across Multiple Collectors</a></span></dt><dt><span class="section"><a href="#_manually_specifying_failover_chains">4.2.2. Manually Specifying Failover Chains</a></span></dt><dt><span class="section"><a href="#_automatic_failover_chains">4.2.3. Automatic Failover Chains</a></span></dt></dl></dd><dt><span class="section"><a href="#_logical_configurations">4.3. Logical Configurations</a></span></dt><dd><dl><dt><span class="section"><a href="#_logical_nodes">4.3.1. Logical Nodes</a></span></dt><dt><span class="section"><a href="#_logical_sources_and_logical_sinks">4.3.2. Logical Sources and Logical Sinks</a></span></dt><dt><span class="section"><a href="#_flow_isolation">4.3.3. Flow Isolation</a></span></dt><dt><span class="section"><a href="#_section_summary_4">4.3.4. Section Summary</a></span></dt></dl></dd><dt><span class="section"><a href="#_multiple_masters">4.4. Multiple Masters</a></span></dt><dd><dl><dt><span class="section"><a href="#_standalone_mode_compared_to_distributed_mode">4.4.1. Standalone Mode Compared to Distributed Mode</a></span></dt><dt><span class="section"><a href="#_running_in_standalone_mode">4.4.2. Running in Standalone Mode</a></span></dt><dt><span class="section"><a href="#_running_in_distributed_mode">4.4.3. Running in Distributed Mode</a></span></dt><dt><span class="section"><a href="#_configuration_stores">4.4.4. Configuration Stores</a></span></dt><dt><span class="section"><a href="#_which_configuration_store_should_i_use">4.4.5. Which Configuration Store Should I Use?</a></span></dt><dt><span class="section"><a href="#_configuring_the_zbcs">4.4.6. Configuring the ZBCS</a></span></dt><dt><span class="section"><a href="#_gossip_in_distributed_mode">4.4.7. Gossip in Distributed Mode</a></span></dt><dt><span class="section"><a href="#_diagrams_how_the_masters_and_nodes_talk_to_each_other">4.4.8. Diagrams: How the Masters and Nodes talk to each other</a></span></dt><dt><span class="section"><a href="#_configuring_flume_nodes_to_connect_to_multiple_master_servers">4.4.9. Configuring Flume Nodes to Connect to Multiple Master Servers</a></span></dt></dl></dd><dt><span class="section"><a href="#_external_zookeeper_cluster">4.5. External ZooKeeper Cluster</a></span></dt><dt><span class="section"><a href="#_section_summary_5">4.6. Section Summary</a></span></dt></dl></div><p>The main goal for Flume is to collect logs and data from many
-different hosts and to scale and intelligently handle different
-cluster and network topologies.</p><p>To deploy Flume on your cluster, do the following steps.</p><div class="itemizedlist" title="Steps to Deploy Flume On a Cluster"><p class="title"><b>Steps to Deploy Flume On a Cluster</b></p><ul class="itemizedlist" type="disc"><li class="listitem">
-Install Flume on each machine.
-</li><li class="listitem">
-Select one or more nodes to be the Master.
-</li><li class="listitem">
-Modify a static configuration file to use site specific properties.
-</li><li class="listitem">
-Start the Flume Master node on at least <span class="strong"><strong>one</strong></span> machine.
-</li><li class="listitem">
-Start a Flume node on <span class="strong"><strong>each</strong></span> machine.
-</li></ul></div><p>The following section describes how to manually configure the
-properties file to specify the Master for each node, and how to set
-default values for parameters.  Sections afterwards describe a data
-flow configuration for a larger system, how to add more capacity by
-adding collectors, and how to improve the reliability of the Master by
-adding multiple Masters.</p><div class="section" title="4.1.�Static Configuration Files"><div class="titlepage"><div><div><h3 class="title"><a name="_static_configuration_files"></a>4.1.�Static Configuration Files</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_using_default_values">4.1.1. Using Default Values</a></span></dt></dl></div><p>In the previous sections, you used Flume on a single machine with the
-default configuration settings. With the default settings, nodes
-automatically search for a Master on <code class="literal">localhost</code> on a standard
-port. In order for the Flume nodes to find the Master in a fully
-distributed setup, you must specify site-specific static configuration
-settings.</p><p>Site-specific settings for Flume nodes and Masters are configured by
-properties in the <code class="literal">conf/flume-site.xml</code> file found on each machine.
-If this file is not present, the commands default to the settings
-found in <code class="literal">conf/flume-conf.xml</code>.  In the following example, you set up
-the property that points a Flume node to search for its Master at a
-machine called <code class="literal">master</code>.</p><p title="conf/flume-site.xml"><b><code class="literal">conf/flume-site.xml</code>.�</b>
-</p><pre class="programlisting">&lt;?xml version="1.0"?&gt;
-&lt;?xml-stylesheet type="text/xsl"  href="configuration.xsl"?&gt;
-
-&lt;configuration&gt;
-&lt;property&gt;
-&lt;name&gt;flume.master.servers&lt;/name&gt;
-&lt;value&gt;master&lt;/value&gt;
-&lt;/property&gt;
-&lt;/configuration&gt;</pre><p title="conf/flume-site.xml">
-</p><div class="section" title="4.1.1.�Using Default Values"><div class="titlepage"><div><div><h4 class="title"><a name="_using_default_values"></a>4.1.1.�Using Default Values</h4></div></div></div><p>When you are using agent/collector roles, you can add the following
-configuration properties to your <code class="literal">flume-site.xml</code> file to set up the
-default hosts used as collector.</p><pre class="programlisting">...
-&lt;property&gt;
-&lt;name&gt;flume.collector.event.host&lt;/name&gt;
-&lt;value&gt;collector&lt;/value&gt;
-&lt;description&gt;This is the host name of the default "remote"     collector.
-&lt;/description&gt;
-&lt;/property&gt;
-  &lt;property&gt;
-&lt;name&gt;flume.collector.port&lt;/name&gt;
-&lt;value&gt;35853&lt;/value&gt;
-&lt;description&gt;This default tcp port that the collector listens to     in order to receive events it is collecting.
-&lt;/description&gt;
-&lt;/property&gt;
-
-...</pre><p>This will make the <code class="literal">agentSink</code> with no arguments default to using
-<code class="literal">flume.collector.event.host</code> and <code class="literal">flume.collector.port</code> for their
-default target and port.</p><p>In the following example, a larger setup with several agents push data
-to a collector. There are seven Flume nodes&#8201;&#8212;&#8201;six in the agent tier,
-and one in the collector tier.</p><div class="informalfigure"><div class="mediaobject"><img src="singleCollector.png" alt="singleCollector.png"></div></div><p>An explicit configuration fills in all of the parameters:</p><pre class="screen">agentA : src | agentSink("collector",35853);
-agentB : src | agentSink("collector",35853);
-agentC : src | agentSink("collector",35853);
-agentD : src | agentSink("collector",35853);
-agentE : src | agentSink("collector",35853);
-agentF : src | agentSink("collector",35853);
-collector : collectorSource(35853) | collectorSink("hdfs://namenode/flume/","srcdata");</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>When specifying destinations for agentSinks, use the <span class="emphasis"><em>hostname
-and port of the target machine</em></span>.  The default name for a node is its
-hostname.  However, if there are multiple logical nodes, you must use
-the machine&#8217;s host name, <span class="emphasis"><em>not the name of the logical node</em></span>. In the
-preceding examples, agent[A-F] and collector are the <span class="emphasis"><em>physical host
-names</em></span> of the machines where these configurations are running.</p></td></tr></table></div><p>You can rely on the default ports set in the configuration files:</p><pre class="screen">agentA : src | agentSink("collector");
-agentB : src | agentSink("collector");
-agentC : src | agentSink("collector");
-agentD : src | agentSink("collector");
-agentE : src | agentSink("collector");
-agentF : src | agentSink("collector");
-collector : collectorSource | collectorSink("hdfs://namenode/flume/","srcdata");</pre><p>You can rely on the default ports and default collector host:</p><pre class="screen">agentA : src | agentSink
-agentB : src | agentSink
-agentC : src | agentSink
-agentD : src | agentSink
-agentE : src | agentSink
-agentF : src | agentSink
-collector : collectorSource | collectorSink("hdfs://namenode/flume/","srcdata");</pre><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>Using defaults can make writing data flow configurations more concise, but may obscure the details about how different nodes are connected to each other.</p></td></tr></table></div><div class="sidebar" title="Reliability Modes"><p class="title"><b>Reliability Modes</b></p><p>You can tune the reliability level of the agents, simply by specifying a different kind of
-agent sink.  There are three levels available, and three corresponding agents:</p><div class="variablelist"><dl><dt><span class="term">
-<code class="literal">agentE2ESink[("<span class="emphasis"><em>machine</em></span>"[,<span class="emphasis"><em>port</em></span>])]</code> 
-</span></dt><dd>
-End to end.  This version uses the WAL, relies on an acknowledgement, and will retry if no acknowledgement is received.
-</dd><dt><span class="term">
-<code class="literal">agentDFOSink[("<span class="emphasis"><em>machine</em></span>"[,<span class="emphasis"><em>port</em></span>])]</code> 
-</span></dt><dd>
-Disk Failover (store on failure).  This agent writes to disk only if it can detect a failure on the collector.  If any data is stored to disk, it periodically retires the network connection and attempts to resend.
-</dd><dt><span class="term">
-<code class="literal">agentBESink[("<span class="emphasis"><em>machine</em></span>"[,<span class="emphasis"><em>port</em></span>])]</code> 
-</span></dt><dd>
-Best Effort.  This agent does not write to disk at all, and drops messages in the event of collector failures.
-</dd></dl></div><p>The previous examples use the <code class="literal">agentSink</code>.  This is an alias for the <code class="literal">agentE2ESink</code>.</p></div></div></div><div class="section" title="4.2.�Multiple Collectors"><div class="titlepage"><div><div><h3 class="title"><a name="_multiple_collectors"></a>4.2.�Multiple Collectors</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_partitioning_agents_across_multiple_collectors">4.2.1. Partitioning Agents across Multiple Collectors</a></span></dt><dt><span class="section"><a href="#_manually_specifying_failover_chains">4.2.2. Manually Specifying Failover Chains</a></span></dt><dt><span class="section"><a href="#_automatic_failover_chains">4.2.3. Automatic Failover Chains</a></span></dt></dl></div><p>Having multiple collectors can increase the log collection throughput
-and can improve the timeliness of event delivery by increasing
-collector availability.  Data collection is parallelizable; thus, load
-from many agents can be shared across many several collectors.</p><div class="section" title="4.2.1.�Partitioning Agents across Multiple Collectors"><div class="titlepage"><div><div><h4 class="title"><a name="_partitioning_agents_across_multiple_collectors"></a>4.2.1.�Partitioning Agents across Multiple Collectors</h4></div></div></div><p>The preceding graph and dataflow spec shows a typical topology for
-Flume nodes.  For reliable delivery, in the event that the collector
-stops operating or disconnects from the agents, the agents would need
-to store their events to their respective disks locally.  The agents
-would then periodically attempt to recontact a collector.  Because the
-collector is down, any analysis or processing downstream is blocked.</p><div class="informalfigure"><div class="mediaobject"><img src="multiCollector.png" alt="multiCollector.png"></div></div><p>When you have multiple collectors as in the preceding graph and
-dataflow spec, downstream progress is still made even in the face of a
-collector&#8217;s failure.  If collector B goes down, agent A, agent B,
-agent E, and agent F continue to deliver events via collector A and
-collector C respectively.  Agent C and agent D may have to queue their
-logs until collector B (or its replacement) comes back online.</p><p>The following configuration partitions the work from the set of agents
-across many collectors.  In this example, each of the collectors
-specify the same DFS output directory and file prefixes, aggregating
-all of the logs into the same directory.</p><pre class="screen">agentA : src | agentE2ESink("collectorA",35853);
-agentB : src | agentE2ESink("collectorA",35853);
-agentC : src | agentE2ESink("collectorB",35853);
-agentD : src | agentE2ESink("collectorB",35853);
-agentE : src | agentE2ESink("collectorC",35853);
-agentF : src | agentE2ESink("collectorC",35853);
-collectorA : collectorSource(35853) | collectorSink("hdfs://...","src");
-collectorB : collectorSource(35853) | collectorSink("hdfs://...","src");
-collectorC : collectorSource(35853) | collectorSink("hdfs://...","src");</pre></div><div class="section" title="4.2.2.�Manually Specifying Failover Chains"><div class="titlepage"><div><div><h4 class="title"><a name="_manually_specifying_failover_chains"></a>4.2.2.�Manually Specifying Failover Chains</h4></div></div></div><div class="informalfigure"><div class="mediaobject"><img src="failoverCollector.png" alt="failoverCollector.png"></div></div><p>When you have multiple collectors writing to the same storage
-location, instead of having agent C and agent D queue indefinitely,
-you can instead have them fail over to other collectors.  In this
-scenario, you could have agent C and agent D fail over to collector A
-and collector C respectively, while periodically checking to see if
-collector B has returned.</p><p>To specify these setups, use agents with <span class="strong"><strong>failover chains</strong></span>. Similarly
-to single collector agents, there are three levels of reliability for
-the failover chain agents: <code class="literal">agentE2EChain</code>, <code class="literal">agentDFOChain</code>, and
-<code class="literal">agentBEChain</code>.</p><p>In the following example, you manually specify the failover chain
-using <code class="literal">agentE2EChain</code>, an agent with end-to-end reliability with
-multiple failover collectors.  <code class="literal">agentA</code> in this situation will
-initially attempt to send to <code class="literal">collectorA</code> on port <code class="literal">35853</code>.  The second
-argument in <code class="literal">agentA</code> 's sink specifies the collector to fall back onto
-if the first fails.  You can specify an arbitrary number of
-collectors, but you must specify at least one.</p><pre class="screen">agentA : src | agentE2EChain("collectorA:35853","collectorB:35853");
-agentB : src | agentE2EChain("collectorA:35853","collectorC:35853");
-agentC : src | agentE2EChain("collectorB:35853","collectorA:35853");
-agentD : src | agentE2EChain("collectorB:35853","collectorC:35853");
-agentE : src | agentE2EChain("collectorC:35853","collectorA:35853");
-agentF : src | agentE2EChain("collectorC:35853","collectorB:35853");
-collectorA : collectorSource(35853) | collectorSink("hdfs://...","src");
-collectorB : collectorSource(35853) | collectorSink("hdfs://...","src");
-collectorC : collectorSource(35853) | collectorSink("hdfs://...","src");</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>In this section, <code class="literal">agent[A-F]</code> and <code class="literal">collector[A-B]</code> are physical host names.</p></td></tr></table></div><p>As in the single collector case, if no port number is specified, the agent defaults to using the <code class="literal">flume.collector.port</code>.</p><pre class="screen">agentA : src | agentE2EChain("collectorA","collectorB");
-agentB : src | agentE2EChain("collectorA","collectorC");
-agentC : src | agentE2EChain("collectorB","collectorA");
-agentD : src | agentE2EChain("collectorB","collectorC");
-agentE : src | agentE2EChain("collectorC","collectorA");
-agentF : src | agentE2EChain("collectorC","collectorB"); collectorA : collectorSource | collectorSink("hdfs://...","src");
-collectorB : collectorSource | collectorSink("hdfs://...","src");
-collectorC : collectorSource | collectorSink("hdfs://...","src");</pre><div class="sidebar" title="Reliability and failover chain semantics"><p class="title"><b>Reliability and failover chain semantics</b></p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-The <code class="literal">agentE2EChain</code> always <span class="emphasis"><em>writes to WAL first</em></span> and then attempts
-  to send to the different collectors.  In the event of a collector
-  failure, it fails over to another collector.
-</li><li class="listitem">
-The <code class="literal">agentDFOChain</code> always attempts to send data to the different
-  collectors, and <span class="emphasis"><em>only writes to disk if all collectors fail</em></span>.
-</li><li class="listitem">
-The <code class="literal">agentBEChain</code> attempts to send message to each collector in
-  succession in the event of failures.  If all collectors fail, it
-  drops messages.
-</li></ul></div></div></div><div class="section" title="4.2.3.�Automatic Failover Chains"><div class="titlepage"><div><div><h4 class="title"><a name="_automatic_failover_chains"></a>4.2.3.�Automatic Failover Chains</h4></div></div></div><p>Flume also provides a mechanism that automatically assigns failover chains based on how nodes are configured.  As collector nodes are assigned in the Flume Master, the Master attempts to distribute the agents evenly amongst the collectors.  In the face of failure, each agent is assigned a different failover chain.  This mitigates the chances of another collector becoming overloaded in the event of failure of a collector.</p><p>To specify a node to use the failover chains, use either the <code class="literal">autoE2EChain</code>, <code class="literal">autoDFOChain</code>, or <code class="literal">autoBEChain</code> agent sink.  Because the Master calculates the failover chains, these sinks take no explicit arguments.</p><pre class="screen">agentA : src | autoE2EChain ;
-agentB : src | autoE2EChain ;
-agentC : src | autoE2EChain ;
-agentD : src | autoE2EChain ;
-agentE : src | autoE2EChain ;
-agentF : src | autoE2EChain ;
-collectorA : autoCollectorSource | collectorSink("hdfs://...", "src");
-collectorB : autoCollectorSource | collectorSink("hdfs://...", "src");
-collectorC : autoCollectorSource | collectorSink("hdfs://...", "src");</pre><p>The Master updates the configuration of the agents based on the current collectors in the system.  When new collectors are added to the system, the Master updates the failover chains of agents to rebalance.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>If the Master has no nodes with <code class="literal">autoCollectorSource</code> as its source, the agent&#8217;s automatic chains will report a <code class="literal">fail("&#8230;")</code> chain which will wait for <code class="literal">autoCollectorSource</code> s to be specified.  If the nodes are not mapped, they will report a different <code class="literal">fail</code> sink notifying you that the node is unmapped (isn&#8217;t associated with a host/port).</p></td></tr></table></div><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>You can see the translation of the auto*Chain configuration in the node configuration table under the translated configuration column.  This is a little declarative specification of the failure recovery behavior of the sink.  More details on this are in the Advanced section of this guide, and in future revisions the translations for the agents and other chains will also be presented.</p></td></tr></table></div></div></div><div class="section" title="4.3.�Logical Configurations"><div class="titlepage"><div><div><h3 class="title"><a name="_logical_configurations"></a>4.3.�Logical Configurations</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_logical_nodes">4.3.1. Logical Nodes</a></span></dt><dt><span class="section"><a href="#_logical_sources_and_logical_sinks">4.3.2. Logical Sources and Logical Sinks</a></span></dt><dt><span class="section"><a href="#_flow_isolation">4.3.3. Flow Isolation</a></span></dt><dt><span class="section"><a href="#_section_summary_4">4.3.4. Section Summary</a></span></dt></dl></div><p>Manually configuring nodes in Flume is manageable for a small number of nodes,
-but can become burdensome for an operator as demands inevitably grow.
-Ideally, the operator only has to assign a role to a particular machine.
-Because configuration management is centralized via the Master, the Master
-potentially has all the information necessary to intelligently create a node
-topology and isolate flows of data from each other.</p><p>To explain how this can be done, the concept of a <span class="strong"><strong>logical node</strong></span> is
-introduced. To manage communications between logical nodes, the concepts of
-<span class="strong"><strong>logical sources</strong></span> and <span class="strong"><strong>logical sinks</strong></span> are introduced.  To isolate different
-groups of nodes, the concept of a <span class="strong"><strong>flow</strong></span> is introduced that allows you to
-group agents and collectors into separate and isolated groups.</p><div class="section" title="4.3.1.�Logical Nodes"><div class="titlepage"><div><div><h4 class="title"><a name="_logical_nodes"></a>4.3.1.�Logical Nodes</h4></div></div></div><p>The logical node abstraction allows for each JVM instance (a physical node) to
-contain multiple logical nodes.  This allows for the processing of many source
-sink combinations on many threads of execution to occur on a single JVM
-instance.</p><p>Each logical node has a name that may be completely different from its
-physical name or hostname.  You now need new operations that enable you to
-spawn a new node, map logical nodes to physical nodes, and decommission
-existing logical nodes.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>The following commands are entered via the web interface using the "raw
-command" web page on the Master.  You might prefer using the Flume command
-shell (described in a later section) for these operations.  The same commands
-described in this section can be entered in web interface or entered at the
-command shell by prefixing the command with <span class="emphasis"><em>exec</em></span>.</p></td></tr></table></div><p>Suppose that initially you know you want an agent-collector topology, but you
-don&#8217;t know the particular names of the exact machines.  For now, you can
-specify the configuration of the logical nodes without specifying any physical
-machine names.</p><pre class="screen">agent1 : _source_ | autoBEChain ;
-collector1 : collectorSource | collectorSink("hdfs://....") ;</pre><p>Later you learn that host1 is the name of the agent1 machine and host2 is the
-name of the collector machine.  You can <span class="emphasis"><em>map</em></span> logical nodes onto the
-physical Flume instances on host1 and host2 by issuing the following map
-commands:</p><pre class="screen">map host1 agent1
-map host2 collector1</pre><p>Afterwards, the node status table should display a new row of information for
-each logical node.  Each logical node reports its own execution state,
-configuration, and heartbeat.  There is also a new entry in the logical node
-mapping table showing that the logical node has been placed on the specified
-physical node. To configure the node&#8217;s sources and sinks, use exactly the same
-mechanisms described in the previous sections.</p><p>You can also remove a logical node by using the <span class="emphasis"><em>decommission</em></span> command.
-Suppose you no longer needed agent1 and wanted to "turn it off".  You can do
-so by entering the following command:</p><pre class="screen">decommission agent1</pre><p>This terminates the thread and removes the configuration associated with a
-logical node, and the mapping between the logical node and physical node.</p><p>You can also move a logical node from one physical node to another by first
-unmapping a logical node and then mapping it on another physical node.  In
-this scenario, you change the collector1 from being on host2 to host3.</p><pre class="screen">unmap host2 collector1</pre><p>At this point, the logical node mapping is removed, and collector1 is not
-active anywhere. You can then map collector1 onto host3 by using the map
-command:</p><pre class="screen">map host3 collector1</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>There are some limitations that need to be further described in this
-section.</p></td></tr></table></div></div><div class="section" title="4.3.2.�Logical Sources and Logical Sinks"><div class="titlepage"><div><div><h4 class="title"><a name="_logical_sources_and_logical_sinks"></a>4.3.2.�Logical Sources and Logical Sinks</h4></div></div></div><p>In the previous example, we used two abstractions under-the-covers that allow
-the specifications of a graph topology for communications <span class="emphasis"><em>without having to
-use physical hostnames and ports</em></span>. These abstractions&#8201;&#8212;&#8201;the <span class="strong"><strong>logical source</strong></span>
-and <span class="strong"><strong>logical sink</strong></span>&#8201;&#8212;&#8201;allow you to create a different graph topology without
-having to know physical machines until they are mapped.</p><p>Suppose you have two nodes producing data and sending it to the consumer:</p><pre class="screen">dataProducer1 : console | logicalSink("dataConsumer") ;
-dataProducer2 : console | logicalSink("dataConsumer") ;
-dataConsumer : logicalSource | console ;</pre><p>Note that in this example, the destination argument is the <span class="strong"><strong>logical name</strong></span> of
-the node and not a specific host/port combination.</p><p>To implement these features, there is a generalized mechanism where users
-enter logical configurations that are <span class="emphasis"><em>translated</em></span> by the Master to a physical
-configuration.</p><p>When the logical nodes get mapped to physical nodes:</p><pre class="screen">map host1 dataProducer1
-map host2 dataProducer2
-map host3 dataConsumer</pre><p>and after the Master learns the host names (the host1, host2, and host3
-machine&#8217;s heartbeat against the Master), the Master has enough information to
-translate configurations with physical hostnames and ports.  A possible
-translation would replace the logicalSource with a rpcSources and the
-logicalSink with an rpcSinks:</p><pre class="screen">dataProducer1 : console | rpcSink("host3",56789) ;
-dataProducer2 : console | rpcSink("host3",56789) ;
-dataConsumer : rpcSource(56789) | console ;</pre><p>In fact, auto agents and collectors, are another example of <span class="strong"><strong>translated
-sources and sinks</strong></span>.  These translate auto*Chain sinks and collectorSource into
-a configuration that uses logicalSinks and logicalSources which in turn are
-translated into physical rpcSource and rpcSinks instances.</p><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>Translations are powerful and can be fairly smart; if new collectors are
-added, they will become new failover options.  If collectors are removed, then
-the removed collectors will be automatically replaced by other failover nodes.</p></td></tr></table></div></div><div class="section" title="4.3.3.�Flow Isolation"><div class="titlepage"><div><div><h4 class="title"><a name="_flow_isolation"></a>4.3.3.�Flow Isolation</h4></div></div></div><p>What happens if you want collect different kinds of data from the same
-physical node?  For example, suppose you wanted to collect httpd logs as well
-as syslog logs from the same physical machine.  Suppose also you want to write
-all of the syslog data from the cluster in one directory tree, and all of the
-httpd logs from the cluster in another.</p><p>One approach is to tag all the data with source information and then push all
-the data down the same pipe.  This could then be followed by some post-
-processing that demultiplexes (demuxes) the data into different buckets.
-Another approach is to keep the two sets of data logically isolated from each
-other the entire time and avoid post processing.</p><p>Flume can do both approaches but enables the latter lower-latency
-approach, by introducing the concept of grouping nodes into
-<span class="strong"><strong>flows</strong></span>. Since there are logical nodes that allow for multiple nodes
-on a single JVM, you can have a node for each different kinds of
-produced data.</p><p>The following example shows how flows can be used. Start by having six logical
-nodes in the system.</p><pre class="screen">fooNode1 : fooSrc | autoBEChain ;
-barNode1 : barSrc | autoBEChain ;
-forNode2 : fooSrc | autoBEChain ;
-barNode2 : barSrc | autoBEChain ;
-fooConsumer : autoCollectorSource | collectorSink("hdfs://nn/foodir") ;
-barConsumer : autoCollectorSource | collectorSink("hdfs://nn/bardir") ;</pre><p>In this scenario, there are two physical machines that produce both
-kinds of data&#8201;&#8212;&#8201;foo data and bar data. You want to send data to
-single collector that collects both foo data and bar data and writes
-it to different HDFS directories.  You could then map the nodes onto
-physical nodes:</p><pre class="screen">map host1 fooNode1
-map host1 barNode1
-map host2 fooNode2
-map host2 barNode2
-map host3 fooConsumer
-map host3 barConsumer</pre><div class="figure"><a name="id415609"></a><p class="title"><b>Figure�1.�Flume Flows: Single Flow</b></p><div class="figure-contents"><div class="mediaobject"><img src="single-flow.png" alt="single-flow.png"></div></div></div><br class="figure-break"><p>This setup essentially instantiates the first approach.  It mixes foo and bar
-data together since the translation of autoBEChain would see two
-collectorSources that the Master considers to be equivalent.  Foo data will
-likely be sent to the barConsumer and bar data will likely be sent to
-fooConsumer.</p><p>You really wanted to separate sources of information into logically isolated
-streams of data.  Flume provides a grouping abstraction called a <span class="strong"><strong>flow</strong></span>.  A
-flow groups particular logical nodes together so that the different logical
-data types remain isolated.</p><p>More concretely, it allows for a different failover chain for each kind of
-data in the Flume cluster.  The auto*Chain based agents would only send data
-to collectors in the same flow group.  This isolates data so that it only
-flows to nodes within the group.</p><p>Currently, the compact form of the configuration language does not
-allow you to specify flows.  Instead you must add an extra argument to
-the config command to specify a flow.</p><p>This example shows commands that would be entered in the Flume shell
-without flow group information.  In this case all of the nodes are in
-the same flow.</p><pre class="screen">exec config fooNode1 fooSrc autoBEChain
-exec config barNode1 barSrc autoBEChain
-exec config fooNode2 fooSrc autoBEChain
-exec config barNode2 barSrc autoBEChain
-exec config fooConsumer autoCollectorSource 'collectorSink("hdfs://nn/foodir")'
-exec config barConsumer autoCollectorSource 'collectorSink("hdfs://nn/bardir")'</pre><p>Now using the following commands you can specify flows by adding an
-extra parameter after the node name.  In this example we have two
-flows: flowfoo and flowbar.  flowfoo contains fooNode1, fooNode2 and
-fooConsumer.  flowbar contains barNode1, barNode2 and barConsumer.</p><pre class="screen">exec config fooNode1 flowfoo fooSrc autoBEChain
-exec config barNode1 flowbar barSrc autoBEChain
-exec config fooNode2 flowfoo fooSrc autoBEChain
-exec config barNode2 flowbar barSrc autoBEChain
-exec config fooConsumer flowfoo autoCollectorSource 'collectorSink("hdfs://nn/foodir")'
-exec config barConsumer flowbar autoCollectorSource 'collectorSink("hdfs://nn/bardir")'</pre><div class="figure"><a name="id415698"></a><p class="title"><b>Figure�2.�Flume Flows: Multiple Flows</b></p><div class="figure-contents"><div class="mediaobject"><img src="multi-flow.png" alt="multi-flow.png"></div></div></div><br class="figure-break"><p>By using these commands, the data from fooNode1 and fooNode2 will only
-be sent to fooConsumer, and barNode1 and barNode2&#8217;s data will only be
-sent to barConsumer.  Data from one node is not mixed with other data
-from other nodes unless explicitly connected.</p><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>In practice it is a good idea to use different node names and
-different flow ids for different kinds of data.  When node names are
-reused, the default behavior is to attempt to recover from failures
-assuming that leftover data from a crashed execution or previous
-source/sink configuration version are still producing the same kind of
-data.</p></td></tr></table></div></div><div class="section" title="4.3.4.�Section Summary"><div class="titlepage"><div><div><h4 class="title"><a name="_section_summary_4"></a>4.3.4.�Section Summary</h4></div></div></div><p>This section introduced logical nodes, logical sources, logical sinks, and
-flows and showed how these abstractions enable you to automatically deal with
-manageability problems.</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Only one input source per physical node.
-</li><li class="listitem">
-Multiple sets of isolated flows.
-</li><li class="listitem">
-Being machine specific, having to know all physical host names and ports.
-</li></ul></div><p>The translation mechanism can be quite powerful.  When coupled with
-metrics information, this could be used to perform automated dynamic
-configuration changes.  A possible example would be to automatically
-commission or decommission new collectors to match diurnal traffic and
-load patterns.</p></div></div><div class="section" title="4.4.�Multiple Masters"><div class="titlepage"><div><div><h3 class="title"><a name="_multiple_masters"></a>4.4.�Multiple Masters</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_standalone_mode_compared_to_distributed_mode">4.4.1. Standalone Mode Compared to Distributed Mode</a></span></dt><dt><span class="section"><a href="#_running_in_standalone_mode">4.4.2. Running in Standalone Mode</a></span></dt><dt><span class="section"><a href="#_running_in_distributed_mode">4.4.3. Running in Distributed Mode</a></span></dt><dt><span class="section"><a href="#_configuration_stores">4.4.4. Configuration Stores</a></span></dt><dt><span class="section"><a href="#_which_configuration_store_should_i_use">4.4.5. Which Configuration Store Should I Use?</a></span></dt><dt><span class="section"><a href="#_configuring_the_zbcs">4.4.6. Configuring the ZBCS</a></span></dt><dt><span class="section"><a href="#_gossip_in_distributed_mode">4.4.7. Gossip in Distributed Mode</a></span></dt><dt><span class="section"><a href="#_diagrams_how_the_masters_and_nodes_talk_to_each_other">4.4.8. Diagrams: How the Masters and Nodes talk to each other</a></span></dt><dt><span class="section"><a href="#_configuring_flume_nodes_to_connect_to_multiple_master_servers">4.4.9. Configuring Flume Nodes to Connect to Multiple Master Servers</a></span></dt></dl></div><p>The Master has two main jobs to perform. The first is to keep track of all the nodes in a Flume deployment and to keep them informed of any changes to their configuration. The second is to track acknowledgements from the end of a Flume flow that is operating in <span class="strong"><strong>reliable mode</strong></span> so that the source at the top of that flow knows when to stop transmitting an event.</p><p>Both these jobs are critical to the operation of a Flume deployment. Therefore, it is ill-advised to have the Master live on a single machine, as this represents a single point of failure for the whole Flume service (see <span class="emphasis"><em>failure modes</em></span> for more detail).</p><p>Flume therefore supports the notion of multiple Masters which run on physically separate nodes and co-ordinate amongst themselves to stay synchronized. If a single Master should fail, the other Masters can take over its duties and keep all live flows functioning. This all happens transparently with a little effort at configuration time. Nodes will automatically fail over to a working Master when they lose contact with their current Master.</p><div class="section" title="4.4.1.�Standalone Mode Compared to Distributed Mode"><div class="titlepage"><div><div><h4 class="title"><a name="_standalone_mode_compared_to_distributed_mode"></a>4.4.1.�Standalone Mode Compared to Distributed Mode</h4></div></div></div><p>The Flume Master can be run in one of two ways.</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-<span class="strong"><strong>Standalone</strong></span> mode - this is where the Master runs on a single machine. This is easy to administer, and simple to set-up, but has disadvantages when it comes to scalability and fault-tolerance.
-</li><li class="listitem">
-<span class="strong"><strong>Distributed</strong></span> mode - this is where the Master is configured to run on several machines - usually three or five. This option scales to serve many Flows, and also has good fault-tolerance properties.
-</li></ul></div><p>Large production deployments of Flume should run a distributed Master so that inevitable machine failures do not impact the availability of Flume itself. For small deployments the issue is less clear-cut - a distributed Master means reserving more computing resources that could be used instead for nodes or other services, and it is possible to recover from many failure modes in a timely manner with human intervention. The choice between distributed and standalone Masters is ultimately dependent both on your use case and your operating requirements.</p></div><div class="section" title="4.4.2.�Running in Standalone Mode"><div class="titlepage"><div><div><h4 class="title"><a name="_running_in_standalone_mode"></a>4.4.2.�Running in Standalone Mode</h4></div></div></div><div class="sidebar"><p class="title"><b></b></p><p>Standalone mode is where Flume has only one Master node. The configuration described below needs to be done on that machine only.</p></div><p>Whether the Flume Master starts in distributed or standalone mode is indirectly controlled by how many machines are configured to run as Master servers. To run in standalone mode, a single configuration property <code class="literal">flume.master.servers</code> must be set:</p><pre class="screen">&lt;property&gt;
-&lt;name&gt;flume.master.servers&lt;/name&gt;
-&lt;value&gt;hostA&lt;/value&gt;
-&lt;/property&gt;</pre><p>The value of <code class="literal">flume.master.servers</code> is a comma-separated list of all the machine names (or IP addresses) that will be Master servers. If this list contains only machine name, the Flume Master will start in standalone mode. If there is more than one machine name in the list, the Flume Master will start in distributed mode.</p><p>There&#8217;s no other configuration required for standalone mode. Flume will use reasonable default values for any other master-related variables. To start the Master, from the command prompt type:</p><pre class="screen">$ flume master</pre><p>from <code class="literal">$FLUME_HOME</code>. A number of log messages should print to the screen. After the server is running, you can check that everything is working properly by visiting the web interface at <code class="literal">http://master-node-ip:35871/</code>,
-where <code class="literal">master-node-ip</code> is the IP address (or hostname) of the Master node. If you see a web page, the Master is running.</p></div><div class="section" title="4.4.3.�Running in Distributed Mode"><div class="titlepage"><div><div><h4 class="title"><a name="_running_in_distributed_mode"></a>4.4.3.�Running in Distributed Mode</h4></div></div></div><div class="sidebar"><p class="title"><b></b></p><p>Distributed mode runs the Flume Master on several machines. Therefore the configuration described below should be done on <span class="strong"><strong>every</strong></span> Master machine, except where noted. <span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span>**</p><p>Running the Flume Master in distributed mode provides better fault tolerance than in standalone mode, and scalability for hundreds of nodes.</p><p>Configuring machines to run as part of a distributed Flume Master is nearly as simple as standalone mode. As before, <code class="literal">flume.master.servers</code> needs to be set, this time to a list of machines:</p><pre class="screen">&lt;property&gt;
-&lt;name&gt;flume.master.servers&lt;/name&gt;
-&lt;value&gt;masterA,masterB,masterC&lt;/value&gt;
-&lt;/property&gt;</pre></div><p title="How many machines do I need?"><b>How many machines do I need?�</b>The distributed Flume Master will continue to work correctly as long as more than half the physical machines running it are still working and haven&#8217;t crashed. Therefore if you want to survive one fault, you need three machines (because 3-1 = 2 &gt; 3/2). For every extra fault you want to tolerate, add another two machines, so for two faults you need five machines. Note that having an even number of machines doesn&#8217;t make the Flume Master any more fault-tolerant - four machines only tolerate one failure, because if two were to fail only two would be left functioning, which is not more than half of four. Common deployments should be well served by three or five machines.</p><div class="sidebar"><p class="title"><b></b></p><p>The final property to set is <span class="strong"><strong>not</strong></span> the same on every machine - every node in the Flume Master must have a unique value for <code class="literal">flume.master.serverid</code>.</p></div><p title="Note"><b>Note.�</b><code class="literal">flume.master.serverid</code> is the only Flume Master property that <span class="emphasis"><em>must</em></span> be different on every machine in the ensemble. <span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong>*</strong></span></p><p title="masterA"><b>masterA.�</b>
-</p><pre class="programlisting">&lt;property&gt;
-&lt;name&gt;flume.master.serverid&lt;/name&gt;
-&lt;value&gt;0&lt;/value&gt;
-&lt;/property&gt;</pre><p title="masterA">
-</p><p title="masterB"><b>masterB.�</b>
-</p><pre class="programlisting">&lt;property&gt;
-&lt;name&gt;flume.master.serverid&lt;/name&gt;
-&lt;value&gt;1&lt;/value&gt;
-&lt;/property&gt;</pre><p title="masterB">
-</p><p title="masterC"><b>masterC.�</b>
-</p><pre class="programlisting">&lt;property&gt;
-&lt;name&gt;flume.master.serverid&lt;/name&gt;
-&lt;value&gt;2&lt;/value&gt;
-&lt;/property&gt;</pre><p title="masterC">
-</p><p>The value for <code class="literal">flume.master.serverid</code> for each node is the index of
-that node&#8217;s hostname in the list in <code class="literal">flume.master.ensemble</code>, starting
-at 0. For example <code class="literal">masterB</code> has index 1 in that list. The purpose of
-this property is to allow each node to uniquely identify itself to the
-other nodes in the Flume Master.</p><p>This is all the configuration required to start a three-node
-distributed Flume Master. To test this out, we can start the Master
-process on all three machines:</p><pre class="screen">[flume@masterA] flume master
-
-[flume@masterB] flume master
-
-[flume@masterC] flume master</pre><p>Each Master process will initially try and contact all other nodes in
-the ensemble. Until more than half (in this case, two) nodes are alive
-and contactable, the configuration store will be unable to start, and
-the Flume Master will not be able to read or write configuration data.</p><p>You can check the current state of the ensemble by inspecting the web page for any of the Flume Master machines which by default will be found at, for example, <code class="literal">http://masterA:35871</code>.</p></div><div class="section" title="4.4.4.�Configuration Stores"><div class="titlepage"><div><div><h4 class="title"><a name="_configuration_stores"></a>4.4.4.�Configuration Stores</h4></div></div></div><p>The Flume Master stores all its data in a <span class="strong"><strong>configuration store</strong></span>. Flume has a pluggable configuration store architecture, and supports two implementations.</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-The Memory-Backed Config Store (MBCS) stores configurations temporarily in memory. If the master node fails and reboots, all the configuration data will be lost. The MBCS is incompatible with distributed masters.  However, it is very easy to administer, computationally lightweight, and good for testing and experimentation.
-</li><li class="listitem">
-The ZooKeeper-Backed Config Store (ZBCS) stores configurations   persistently and takes care of synchronizing them between multiple masters.
-</li></ul></div><p title="Flume and Apache ZooKeeper"><b>Flume and Apache ZooKeeper <span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span>.�</b>Flume relies on the Apache ZooKeeper coordination platform to provide reliable, consistent, and persistent storage for node configuration data. A ZooKeeper ensemble is made up of two or more nodes which communicate regularly with each other to make sure each is up to date. Flume embeds a ZooKeeper server inside the Master process, so starting and maintaining the service is taken care of. However, if you have an existing ZooKeeper service running, Flume supports using that external cluster as well. <span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span><span class="strong"><strong><span class="strong"><strong></strong></span></strong></span></p></div><div class="section" title="4.4.5.�Which Configuration Store Should I Use?"><div class="titlepage"><div><div><h4 class="title"><a name="_which_configuration_store_should_i_use"></a>4.4.5.�Which Configuration Store Should I Use?</h4></div></div></div><p>In almost all cases, you should use the ZBCS. It is more reliable and fault-tolerant, and will recover configurations after a restart. It is compatible with both standalone and distributed deployments of the Flume Master.</p><p>The MBCS is appropriate if you are experimenting with Flume and can stand to lose configuration if the machine fails.</p><p>ZBCS is the default configuration store. The choice of which configuration store to use is controlled by the <code class="literal">flume.master.store</code> system property.</p><pre class="programlisting">&lt;property&gt;
-&lt;name&gt;flume.master.store&lt;/name&gt;
-&lt;value&gt;zookeeper&lt;/value&gt;
-&lt;/property&gt;</pre><p>If set to <code class="literal">memory</code>, the Flume Master will use MBCS instead. This is only supported in standalone mode.</p></div><div class="section" title="4.4.6.�Configuring the ZBCS"><div class="titlepage"><div><div><h4 class="title"><a name="_configuring_the_zbcs"></a>4.4.6.�Configuring the ZBCS</h4></div></div></div><p>Most deployments using the ZBCS can use Flume&#8217;s default configuration. However, where more control over the precise configuration of the Flume Master is needed, there are several properties that you can set.</p><div class="variablelist"><dl><dt><span class="term">
-Log Directory - <code class="literal">flume.master.zk.logdir</code> 
-</span></dt><dd>
-To ensure reliability and the ability to restore its state in the event of a failure, ZBCS continually logs all updates it sees to the directory in <code class="literal">flume.master.zk.logdir</code>. This directory must be writable by the user as which Flume is running, and will be created if it doesn&#8217;t exist at start-up time. WARNING: Do not delete this directory, or any files inside it. If deleted, all your configuration information will be lost.
-</dd><dt><span class="term">
-ZBCS Server Ports 
-</span></dt><dd>
-Each machine in the distributed Flume Master communicates with every other on the TCP ports set by <code class="literal">flume.master.zk.server.quorum.port</code> and <code class="literal">flume.master.zk.server.election.port</code>. The defaults are 3182 and 3183 respectively. Note that these settings control both the port on which the ZBCS listens, and on which it looks for other machines in the ensemble.
-</dd><dt><span class="term">
-ZBCS Client Port - <code class="literal">flume.master.zk.client.port</code> 
-</span></dt><dd>
-The Flume Master process communicates with ZooKeeper (either on the same machine, or remotely on another Master server) via a client TCP port, which is set by <code class="literal">flume.master.zk.client.port</code>. The default is 3181.
-</dd></dl></div></div><div class="section" title="4.4.7.�Gossip in Distributed Mode"><div class="titlepage"><div><div><h4 class="title"><a name="_gossip_in_distributed_mode"></a>4.4.7.�Gossip in Distributed Mode</h4></div></div></div><p>Flume Master servers also use a <span class="emphasis"><em>gossip</em></span> protocol to exchange information between themselves. Each server periodically wakes and picks another machine to send new data to. This protocol by default uses TCP port 57890, but this is controlled via the <code class="literal">flume.master.gossipport</code> property:</p><pre class="screen">&lt;property&gt;
-&lt;name&gt;flume.master.gossip.port&lt;/name&gt;
-&lt;value&gt;57890&lt;/value&gt;
-&lt;/property&gt;</pre><p>In standalone mode, there is no need to use gossip, so this port is unused.</p></div><div class="section" title="4.4.8.�Diagrams: How the Masters and Nodes talk to each other"><div class="titlepage"><div><div><h4 class="title"><a name="_diagrams_how_the_masters_and_nodes_talk_to_each_other"></a>4.4.8.�Diagrams: How the Masters and Nodes talk to each other</h4></div></div></div><div class="figure"><a name="id416532"></a><p class="title"><b>Figure�3.�Flume Master: Standalone Mode</b></p><div class="figure-contents"><div class="mediaobject"><img src="master-zk-standalone.png" alt="master-zk-standalone.png"></div></div></div><br class="figure-break"><div class="figure"><a name="id416555"></a><p class="title"><b>Figure�4.�Flume Master: Distributed Mode</b></p><div class="figure-contents"><div class="mediaobject"><img src="master-zk-internal.png" alt="master-zk-internal.png"></div></div></div><br class="figure-break"></div><div class="section" title="4.4.9.�Configuring Flume Nodes to Connect to Multiple Master Servers"><div class="titlepage"><div><div><h4 class="title"><a name="_configuring_flume_nodes_to_connect_to_multiple_master_servers"></a>4.4.9.�Configuring Flume Nodes to Connect to Multiple Master Servers</h4></div></div></div><p>One property needs to be set to configure a Flume Node to connect to multiple Masters: <code class="literal">flume.master.servers</code>.</p><pre class="programlisting">&lt;property&gt;
-&lt;name&gt;flume.master.servers&lt;/name&gt;
-&lt;value&gt;masterA,masterB,masterC&lt;/value&gt;
-&lt;/property&gt;</pre><p>The nodes connect over the port <code class="literal">flume.master.heartbeat.port</code> on each machine in the Flume Master - this is the port that the Master servers listen on for node heartbeats.</p><p>If a Master server fails, nodes will automatically fail over to the next randomly selected Master server that they can establish a connection to.</p></div></div><div class="section" title="4.5.�External ZooKeeper Cluster"><div class="titlepage"><div><div><h3 class="title"><a name="_external_zookeeper_cluster"></a>4.5.�External ZooKeeper Cluster</h3></div></div></div><p>In some cases you may want a ZBCS that relies on an externally managed ZooKeeper service. The most common example of this is where multiple services which rely on ZooKeeper are being used (Flume and Hbase for example). In the following example zkServer{A,B,C}:2181 should be replaced with the hostname/port of the ZooKeeper servers which make up your ensemble.</p><p title="conf/flume-site.xml"><b><code class="literal">conf/flume-site.xml</code>.�</b>
-</p><pre class="programlisting">&lt;property&gt;
-  &lt;name&gt;flume.master.zk.use.external&lt;/name&gt;
-  &lt;value&gt;true&lt;/value&gt;
-&lt;/property&gt;
-
-&lt;property&gt;
-  &lt;name&gt;flume.master.zk.servers&lt;/name&gt;
-  &lt;value&gt;zkServerA:2181,zkServerB:2181,zkServerC:2181&lt;/value&gt;
-&lt;/property&gt;</pre><p title="conf/flume-site.xml">
-</p></div><div class="section" title="4.6.�Section Summary"><div class="titlepage"><div><div><h3 class="title"><a name="_section_summary_5"></a>4.6.�Section Summary</h3></div></div></div><p>This section described installing, deploying, and configuring a set of
-Flume nodes in a fully distributed setting. You should now be able to
-collect streams of logs with Flume.</p><p>You also used some roles for sources and sinks to connect nodes
-together.  You now have an understanding of the basics of setting up a
-set of Flume nodes.  Here&#8217;s the new sources and sinks introduced in
-this subsection.</p><div class="variablelist" title="Flume&#8217;s Tiered Event Sources"><p class="title"><b>Flume&#8217;s Tiered Event Sources</b></p><dl><dt><span class="term">
-<code class="literal">collectorSource[(<span class="emphasis"><em>port</em></span>)]</code> 
-</span></dt><dd>
-Collector source. Listens for data from
-agentSinks forwarding to port <code class="literal"><span class="emphasis"><em>port</em></span></code>.  If port is not specified, the
-node default collector TCP port, 35853.
-</dd></dl></div></div></div><div class="section" title="5.�Integrating Flume with your Data Sources"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_integrating_flume_with_your_data_sources"></a>5.�Integrating Flume with your Data Sources</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_push_sources">5.1. Push Sources</a></span></dt><dt><span class="section"><a href="#_polling_sources">5.2. Polling Sources</a></span></dt><dt><span class="section"><a href="#_embedding_sources">5.3. Embedding Sources</a></span></dt><dt><span class="section"><a href="#_logging_via_log4j_directly">5.4. Logging via log4j Directly</a></span></dt><dd><dl><dt><span class="section"><a href="#_example_of_logging_hadoop_jobs">5.4.1. Example of Logging Hadoop Jobs</a></span></dt><dt><span class="section"><a href="#_logging_hadoop_daemons">5.4.2. Logging Hadoop Daemons</a></span></dt></dl></dd></dl></div><div class="sidebar" title="WARNING"><p class="title"><b>WARNING</b></p><p>This section is incomplete.</p></div><p>Flume&#8217;s source interface is designed to be simple yet powerful and enable logging
-of all kinds of data&#8201;&#8212;&#8201;from unstructured blobs of byte, semi-structured blobs
-with structured metadata, to completely structured data.</p><p>In this section we describe some of the basic mechanisms that can be used to
-pull in data.  Generally, this approach has three flavors. <span class="strong"><strong>Pushing</strong></span> data to
-Flume, having Flume <span class="strong"><strong>polling</strong></span> for data, or <span class="strong"><strong>embedding</strong></span> Flume or Flume
-components into an application.</p><p>These mechanisms have different trade-offs&#8201;&#8212;&#8201;based on the semantics of the
-operation.</p><p>Also, some sources can be <span class="strong"><strong>one shot</strong></span> or <span class="strong"><strong>continuous</strong></span> sources.</p><div class="section" title="5.1.�Push Sources"><div class="titlepage"><div><div><h3 class="title"><a name="_push_sources"></a>5.1.�Push Sources</h3></div></div></div><div class="variablelist"><dl><dt><span class="term">
-<code class="literal">syslogTcp</code>, <code class="literal">syslogUdp</code> 
-</span></dt><dd>
-wire-compatibility with syslog, and syslog-ng
-logging protocols.
-</dd><dt><span class="term">
-<code class="literal">scribe</code> 
-</span></dt><dd>
-wire-compatibility with the scribe log collection system.
-</dd></dl></div></div><div class="section" title="5.2.�Polling Sources"><div class="titlepage"><div><div><h3 class="title"><a name="_polling_sources"></a>5.2.�Polling Sources</h3></div></div></div><div class="variablelist"><dl><dt><span class="term">
-<code class="literal">tail</code>, <code class="literal">multitail</code> 
-</span></dt><dd>
-watches a file(s) for appends.
-</dd><dt><span class="term">
-<code class="literal">exec</code> 
-</span></dt><dd>
-This is good for extracting custom data by using existing programs.
-</dd><dt><span class="term">
-<code class="literal">poller</code> 
-</span></dt><dd>
-We can gather information from Flume nodes themselves.
-</dd></dl></div></div><div class="section" title="5.3.�Embedding Sources"><div class="titlepage"><div><div><h3 class="title"><a name="_embedding_sources"></a>5.3.�Embedding Sources</h3></div></div></div><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>these features are incomplete.</p></td></tr></table></div><p><code class="literal">log4j</code></p><p><code class="literal">simple client library</code></p><pre class="literallayout">// move this to gathering data from sources</pre></div><div class="section" title="5.4.�Logging via log4j Directly"><div class="titlepage"><div><div><h3 class="title"><a name="_logging_via_log4j_directly"></a>5.4.�Logging via log4j Directly</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_example_of_logging_hadoop_jobs">5.4.1. Example of Logging Hadoop Jobs</a></span></dt><dt><span class="section"><a href="#_logging_hadoop_daemons">5.4.2. Logging Hadoop Daemons</a></span></dt></dl></div><div class="section" title="5.4.1.�Example of Logging Hadoop Jobs"><div class="titlepage"><div><div><h4 class="title"><a name="_example_of_logging_hadoop_jobs"></a>5.4.1.�Example of Logging Hadoop Jobs</h4></div></div></div></div><div class="section" title="5.4.2.�Logging Hadoop Daemons"><div class="titlepage"><div><div><h4 class="title"><a name="_logging_hadoop_daemons"></a>5.4.2.�Logging Hadoop Daemons</h4></div></div></div></div></div></div><div class="section" title="6.�Using Data Collected by Flume"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_using_data_collected_by_flume"></a>6.�Using Data Collected by Flume</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_the_data_model_of_a_flume_event">6.1. The Data Model of a Flume Event</a></span></dt><dt><span class="section"><a href="#_output_bucketing">6.2. Output Bucketing</a></span></dt><dt><span class="section"><a href="#_output_format">6.3. Output Format</a></span></dt><dt><span class="section"><a href="#_small_files_compared_to_high_latency">6.4. Small Files Compared to High Latency</a></span></dt></dl></div><p>The first goal of Flume is to collect data and reliably write it to HDFS.
-Once data arrives, one wants the ability to control where and in what format
-data is stored.  Flume provides basic output control mechanisms via the
-properties configuration and in the dataflow language.  This gives the user
-the ability to control the output format and output bucketing of incoming
-data, and simplifies integration with other HDFS data consumers such as Hive
-and HBase.</p><p>Here are some example use cases:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-When monitoring a web server, you want to bucket logs based on time,
-  the page hit, and the browser being used.
-</li><li class="listitem">
-When tracking particular data nodes, you want to bucket logs based on
-  time and the data node name.
-</li><li class="listitem">
-When tracking a feed of JIRA tickets from the Apache feed, you want
-  to group based on the project identifier or a particular person.
-</li><li class="listitem">
-When collecting data from scribe sources, you want to use its bucket
-  data based on its the event&#8217;s category information.
-</li></ul></div><p>To support these kinds of features, Flume uses a simple data model,
-provides a mechanism for bucketing events, and also provides basic
-extraction operations for specifying custom bucketing discriminators.</p><div class="section" title="6.1.�The Data Model of a Flume Event"><div class="titlepage"><div><div><h3 class="title"><a name="_the_data_model_of_a_flume_event"></a>6.1.�The Data Model of a Flume Event</h3></div></div></div><p>A Flume event has these six main fields:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Unix timestamp
-</li><li class="listitem">
-Nanosecond timestamp
-</li><li class="listitem">
-Priority
-</li><li class="listitem">
-Source host
-</li><li class="listitem">
-Body
-</li><li class="listitem">
-Metadata table with an arbitrary number of attribute value pairs.
-</li></ul></div><p>All events are guaranteed to have all of these elements.  However, the
-body may have zero length, and the metadata table can be empty.</p><p>The Unix timestamp is measured in milliseconds and is Unix time stamp from the
-source machine.  The nanosecond timestamp is machine specific nanosecond
-counter also from the source machine.  It is safe to assume that the nanotime
-from a machine is monotonically increasing&#8201;&#8212;&#8201;i.e. if event A has a larger
-nanotime than event B from the same machine, event A was initially received
-before event B.</p><p>Currently the priority of a message can have one of 6 values: TRACE, DEBUG,
-INFO, WARN, ERROR, or FATAL.  These values are often provided by logging
-systems such as syslog or log4j.</p><p>The source host is the name of the machine or the IP (whatever hostname call
-returns).</p><p>The body is the raw log entry body.  The default is to truncate the body to a
-maximum of 32KB per event.  This is a configurable value and can be changed by
-modifying the <span class="emphasis"><em>flume.event.max.size.bytes</em></span> property.</p><p>Finally there is the metadata table which is a map from a string attribute
-name to an arbitrary array of bytes.  This allows for custom bucketing
-attributes and will be described in more depth in the Advanced Usage section
-of this guide.</p></div><div class="section" title="6.2.�Output Bucketing"><div class="titlepage"><div><div><h3 class="title"><a name="_output_bucketing"></a>6.2.�Output Bucketing</h3></div></div></div><p>You can control the output of events to particular directories or files based
-on the values of an event&#8217;s fields.  To enable this, you provide an escaping
-mechanism that outputs data to a particular path.</p><p>For example, here is an output spec:</p><pre class="screen">collectorSink("hdfs://namenode/flume/webdata/%H00/", "%{host}-")</pre><p>The first argument is the directory where data is to be written.  The second
-is a filename prefix where events are written.  Suppose you get an event from a
-machine called server1 generated at time 18:58.  The events would get written
-to HDFS with namenode namenode, in a directory called /flume/webdata/1800/,
-with files named server1-xxx where xxx is some extra data for unique file
-names.</p><p>What happened here?  Flume replaced the <span class="emphasis"><em>%H</em></span> with a string that represent the
-hour of the timestamp found in the event&#8217;s data. Likewise, the <span class="emphasis"><em>%o</em></span> was
-replace with the hostname field from the event.</p><p>What happens if the server1&#8217;s message had been delayed and the message wasn&#8217;t
-sent downstream until 19:05?  Since the value of the timestamp on the event
-was during the 18:00 hour, the event would be written into that directory.</p><div class="variablelist" title="Event data escape sequences"><p class="title"><b>Event data escape sequences</b></p><dl><dt><span class="term">
-[horizontal] %{host} 
-</span></dt><dd>
-host
-</dd><dt><span class="term">
-%{nanos} 
-</span></dt><dd>
-nanos
-</dd><dt><span class="term">
-%{priority} 
-</span></dt><dd>
-priority string
-</dd><dt><span class="term">
-%{body} 
-</span></dt><dd>
-body
-</dd><dt><span class="term">
-%% 
-</span></dt><dd>
-a <span class="emphasis"><em>%</em></span> character.
-</dd><dt><span class="term">
-%t 
-</span></dt><dd>
-Unix time in millis
-</dd></dl></div><p>Because bucketing by date is a frequently-requested feature, there are escape
-sequences for finer control of date values that allow you to bucket data based
-on date.</p><p>Here is another output spec:</p><pre class="screen">collectorSink("hdfs://namenode/flume/webdata/%Y-%m-%d/%H00/", "web-")</pre><p>This would create directories for each day, each with a subdirectory for each
-hour with filenames prefixed "web-".</p><div class="horizontal"><a name="id417284"></a><p class="title"><b>Table�2.�Fine grained escape sequences date and times</b></p><div class="horizontal-contents"><table summary="Fine grained escape sequences date and times" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-%a 
-</p>
-</td><td style="" valign="top">
-<p>
-locale&#8217;s short weekday name (Mon, Tue, &#8230;)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%A 
-</p>
-</td><td style="" valign="top">
-<p>
-locale&#8217;s full weekday name (Monday, Tuesday, &#8230;)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%b 
-</p>
-</td><td style="" valign="top">
-<p>
-locale&#8217;s short month name (Jan, Feb,&#8230;)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%B 
-</p>
-</td><td style="" valign="top">
-<p>
-locale&#8217;s long month name (January, February,&#8230;)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%c 
-</p>
-</td><td style="" valign="top">
-<p>
-locale&#8217;s date and time (Thu Mar  3 23:05:25 2005)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%d 
-</p>
-</td><td style="" valign="top">
-<p>
-day of month (01)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%D 
-</p>
-</td><td style="" valign="top">
-<p>
-date; same as %m/%d/%y
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%H 
-</p>
-</td><td style="" valign="top">
-<p>
-hour (00..23)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%I 
-</p>
-</td><td style="" valign="top">
-<p>
-hour (01..12)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%j 
-</p>
-</td><td style="" valign="top">
-<p>
-day of year (001..366)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%k 
-</p>
-</td><td style="" valign="top">
-<p>
-hour ( 0..23)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%l 
-</p>
-</td><td style="" valign="top">
-<p>
-hour ( 1..12)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%m 
-</p>
-</td><td style="" valign="top">
-<p>
-month (01..12)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%M 
-</p>
-</td><td style="" valign="top">
-<p>
-minute (00..59)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%P 
-</p>
-</td><td style="" valign="top">
-<p>
-locale&#8217;s equivalent of am or pm
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%s 
-</p>
-</td><td style="" valign="top">
-<p>
-seconds since 1970-01-01 00:00:00 UTC
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%S 
-</p>
-</td><td style="" valign="top">
-<p>
-second (00..60)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%y 
-</p>
-</td><td style="" valign="top">
-<p>
-last two digits of year (00..99)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%Y 
-</p>
-</td><td style="" valign="top">
-<p>
-year (2010)
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-%z 
-</p>
-</td><td style="" valign="top">
-<p>
-+hhmm numeric timezone (for example, -0400)
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"></div><div class="section" title="6.3.�Output Format"><div class="titlepage"><div><div><h3 class="title"><a name="_output_format"></a>6.3.�Output Format</h3></div></div></div><p>Now that you have control of where files go, this section describes how you can
-control the output format of data.  Currently, this is set via the
-<span class="emphasis"><em>flume.collector.output.format</em></span> property set in the flume-site.xml file.  The output
-formats are:</p><div class="horizontal"><a name="id417657"></a><p class="title"><b>Table�3.�Output formats</b></p><div class="horizontal-contents"><table summary="Output formats" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-syslog 
-</p>
-</td><td style="" valign="top">
-<p>
-a syslog like text output format
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-log4j 
-</p>
-</td><td style="" valign="top">
-<p>
-a log4j pattern similar to that used by CDH output pattern.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-raw 
-</p>
-</td><td style="" valign="top">
-<p>
-Event body only.  This is most similar to copying a file but
-does not preserve any uniqifying metadata like host/timestamp/nanos.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-avro 
-</p>
-</td><td style="" valign="top">
-<p>
-Avro Native file format.  Default currently is uncompressed.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-avrojson 
-</p>
-</td><td style="" valign="top">
-<p>
-JSON encoded date generated by avro
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-avrodata 
-</p>
-</td><td style="" valign="top">
-<p>
-Binary encoded data written in the avro binary format.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-default 
-</p>
-</td><td style="" valign="top">
-<p>
-a debugging format.
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"><pre class="screen">&lt;property&gt;
-  &lt;name&gt;flume.collector.output.format&lt;/name&gt;
-  &lt;value&gt;avrojson&lt;/value&gt;
-  &lt;description&gt;This is the output format for the data written to the
-  collector.  There are several formats available:
-    syslog - outputs events in a syslog-like format
-    log4j - outputs events in a pattern similar to Hadoop's log4j pattern
-    raw - Event body only.  This is most similar to copying a file but
-      does not preserve any uniqifying metadata like host/timestamp/nanos.
-    avro - Avro Native file format.  Default currently is uncompressed.
-    avrojson - this outputs data as json encoded by avro
-    avrodata - this outputs data as an avro binary encoded data
-    debug - this is a format for debugging
-  &lt;/description&gt;
-&lt;/property&gt;</pre></div><div class="section" title="6.4.�Small Files Compared to High Latency"><div class="titlepage"><div><div><h3 class="title"><a name="_small_files_compared_to_high_latency"></a>6.4.�Small Files Compared to High Latency</h3></div></div></div><p>For all versions Hadoop&#8217;s file system that are earlier than 0.20.x,
-HDFS has write-once read-many semantics. Thus, the only way to
-reliably flush an HDFS file is to close the file.  Moreover, once a
-file is closed, no new data can be appended to the file.  This
-presents a tension between getting data written quickly to HDFS and
-potentially having many small files (which is a potential scalability
-bottleneck of HDFS).</p><p>On one side, to minimize the load and data stored throughput the system,
-ideally one would flush data to HDFS as soon as it arrives. Flushing
-frequently is in conflict with efficiently storing data to HDFS because this
-could result in many small files, which eventually will stress an HDFS
-namenode.  A compromise is to pick a reasonable trigger that has a collector
-close "reasonably-sized" files (ideally larger than a single HDFS block, 64MB
-by default).</p><p>When Flume is deployed at a scale where data collection volumes are small, it
-may take a long time to reach the ideal minimum file size (a block size,
-typically 64MB).  For example, if a single web server produces 10k of logs a
-second (approx. 100 hit logs/s at 100B per log), it will take about 2 hours
-(6400 seconds) before an ideal file size can reached.</p><p>In these situations, lean towards having more small files.  Small files
-cause a few problems downstream.  These include potential scaling limitations
-of Hadoop&#8217;s HDFS, and performance penalties when using MapReduce&#8217;s default
-input processing mechanisms within Hadoop.</p><p>The following sections describe two mechanisms to mitigate these
-potential problems:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Rolling up many small data files into larger batches
-</li><li class="listitem">
-Using a CombinedFileInputFormat
-</li></ul></div><p>This particular problem becomes less of an issue when the scale of logging
-goes up.  If a hundred machines were generating the same amount of logs, you
-would reach reasonable files sizes every 64 seconds.</p><p>Future versions of Hadoop will mitigate this problem by providing a flush/sync
-operation for currently open HDFS files (patch is already slated for
-Hadoop HDFS 0.21.x).</p></div></div><div class="section" title="7.�Compression for files written to HDFS."><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_compression_for_files_written_to_hdfs"></a>7.�Compression for files written to HDFS.</h2></div></div></div><p>Flume supports basic compression for all log files that are written to
-HDFS.  Compressed files are automatically suffixed with an extension
-and follow the same naming format + directory structure as regular log
-files.</p><p>If GZipCodec is selected, ".gz" is appended to the file name, if
-BZip2Codec is selected, ".bz2" is appended.</p><pre class="screen">  &lt;property&gt;
-    &lt;name&gt;flume.collector.dfs.compress.codec&lt;/name&gt;
-    &lt;value&gt;None&lt;/value&gt;
-    &lt;description&gt;Writes formatted data compressed in specified codec to
-    dfs. Value is None, GZipCodec, DefaultCodec (deflate), BZip2Codec,
-    or any other Codec Hadoop is aware of &lt;/description&gt;
-  &lt;/property&gt;</pre></div><div class="section" title="8.�Advanced Flume Usage"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_advanced_flume_usage"></a>8.�Advanced Flume Usage</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_the_flume_command_shell">8.1. The Flume Command Shell</a></span></dt><dd><dl><dt><span class="section"><a href="#_using_the_flume_command_shell">8.1.1. Using the Flume Command Shell</a></span></dt></dl></dd><dt><span class="section"><a href="#_flume_8217_s_dataflow_specification_language">8.2. Flume&#8217;s Dataflow Specification Language</a></span></dt><dd><dl><dt><span class="section"><a href="#_special_sinks_fan_out_fail_over_and_roll">8.2.1. Special Sinks: Fan out, Fail over, and Roll</a></span></dt><dt><span class="section"><a href="#_introducing_sink_decorators">8.2.2. Introducing Sink Decorators</a></span></dt><dt><span class="section"><a href="#_translations_of_high_level_sources_and_sinks">8.2.3. Translations of High-level Sources and Sinks</a></span></dt></dl></dd><dt><span class="section"><a href="#_custom_metadata_extraction">8.3. Custom Metadata Extraction</a></span></dt><dd><dl><dt><span class="section"><a href="#_extractors">8.3.1. Extractors</a></span></dt><dt><span class="section"><a href="#_meta_data_filtering_and_transformations">8.3.2. Meta Data Filtering and Transformations</a></span></dt><dt><span class="section"><a href="#_role_defaults">8.3.3. Role Defaults</a></span></dt><dt><span class="section"><a href="#_arbitrary_data_flows_and_custom_architectures">8.3.4. Arbitrary Data Flows and Custom Architectures</a></span></dt></dl></dd><dt><span class="section"><a href="#_extending_via_sink_source_decorator_plugins">8.4. Extending via Sink/Source/Decorator Plugins</a></span></dt><dd><dl><dt><span class="section"><a href="#_semantics_of_flume_extensions">8.4.1. Semantics of Flume Extensions</a></span></dt><dd><dl><dt><span class="section"><a href="#_simple_source_semantics">8.4.1.1. Simple source semantics.</a></span></dt><dt><span class="section"><a href="#_buffered_source_semantics">8.4.1.2. Buffered source semantics</a></span></dt><dt><span class="section"><a href="#_simple_sinks">8.4.1.3. Simple Sinks.</a></span></dt><dt><span class="section"><a href="#_buffered_sink_and_decorator_semantics">8.4.1.4. Buffered sink and decorator semantics</a></span></dt><dt><span class="section"><a href="#_retries_sleeps_and_unclean_exits">8.4.1.5. Retries, sleeps, and unclean exits.</a></span></dt></dl></dd></dl></dd><dt><span class="section"><a href="#_limiting_data_transfer_rate_between_source_sink_pairs">8.5. Limiting Data Transfer Rate between Source-Sink pairs</a></span></dt></dl></div><p>This section describes in further detail, how to automate the
-control of Flume nodes via the FlumeShell, a deep dive into Flume&#8217;s dataflow
-specification language, internals of the reliability mechanisms, how to do
-metadata manipulations, and how to install source and sink plugins.</p><div class="section" title="8.1.�The Flume Command Shell"><div class="titlepage"><div><div><h3 class="title"><a name="_the_flume_command_shell"></a>8.1.�The Flume Command Shell</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_using_the_flume_command_shell">8.1.1. Using the Flume Command Shell</a></span></dt></dl></div><p>So far, you have been modifying the state of Flume using a simple (but
-primitive) web interface to a Master server.</p><p>Flume also provides a shell, which allows the user to type commands into a
-terminal and have them executed on a Flume deployment.</p><p>All of the commands available in the web form are available in the
-Flume Shell.  The Flume Shell, however, actually has extra controls
-for command submission control and state checking that aid
-scriptability.</p><div class="section" title="8.1.1.�Using the Flume Command Shell"><div class="titlepage"><div><div><h4 class="title"><a name="_using_the_flume_command_shell"></a>8.1.1.�Using the Flume Command Shell</h4></div></div></div><p>You can start the FlumeShell by running <code class="literal">flume shell</code> in a terminal window.
-The <code class="literal">connect</code> command can be used to establish a connection to any Master
-server.</p><pre class="screen">hostname:~/flume$ flume shell
-[flume (disconnected)] connect localhost:35873
-Connecting to Flume master localhost:35873...
-[flume localhost:35873]</pre><pre class="screen">hostname:~/flume$ flume shell -c localhost:35873
-Connecting to Flume master localhost:35873...
-[flume localhost:35873]</pre><div class="sidebar"><p class="title"><b></b></p><p>The port to use is the value of <code class="literal">flume.config.admin.port</code> and defaults to 35873.</p></div><p>The command line parameters for the Flume Shell are as follows:</p><pre class="screen">usage: FlumeShell [-c &lt;arg&gt;] [-e &lt;arg&gt;] [-q] [-s &lt;arg&gt;]
- -?         Command line usage help
- -c &lt;arg&gt;   Connect to master:port
- -e &lt;arg&gt;   Run a single command
- -q         Run in quiet mode - only print command results
- -s &lt;arg&gt;   Run a FlumeShell script</pre><p>The FlumeShell makes scripting Flume possible - either by using a single
-invocation with <code class="literal">-e</code> or by running a script of commands with <code class="literal">-s</code>. It is also
-possible to pipe <code class="literal">stdin</code> to the FlumeShell as in the following example:</p><pre class="screen">echo "connect localhost:35873\ngetconfigs\nquit" | flume shell -q</pre><p title="Flume Commands"><b>Flume Commands.�</b>You can press Tab any time for some hints on available commands.  If you start
-typing a command you can use TAB to complete command.</p><div class="variablelist"><dl><dt><span class="term">
-<code class="literal">help</code> 
-</span></dt><dd>
-List the commands available in the shell.
-</dd><dt><span class="term">
-<code class="literal">connect <span class="emphasis"><em>master:port</em></span></code> 
-</span></dt><dd>
-connect to a master at machine <span class="emphasis"><em>master</em></span> on port
-<span class="emphasis"><em>port</em></span>.
-</dd><dt><span class="term">
-<code class="literal">config</code> <span class="emphasis"><em>logicalnode</em></span> <span class="emphasis"><em>source</em></span> <span class="emphasis"><em>sink</em></span> 
-</span></dt><dd>
-configure a single logical node
-<span class="emphasis"><em>logicalnode</em></span> with source <span class="emphasis"><em>source</em></span> and sink <span class="emphasis"><em>sink</em></span>.  <span class="emphasis"><em>source</em></span> and <span class="emphasis"><em>sink</em></span> will
-likely need quotes to support some of the Flume configuration syntax.
-</dd><dt><span class="term">
-<code class="literal">getnodestatus</code> 
-</span></dt><dd>
-Output the status of the nodes the master knows
-about. Nodes are in either HELLO, CONFIGURING, ACTIVE, IDLE, ERROR,
-DECOMMISSIONED, or LOST states.  When a node shows up initially it is
-HELLO state.  When a node is being configured, it is in CONFIGURING
-state.  Once events are being pumped from source to sink, the node is
-in ACTIVE state.  If a node has drained its source (and the source is
-not "endless") it will enter IDLE state.  If a node encountered an
-unrecoverable error or exited without flushing, it will be in ERROR
-state.  A node is DECOMMISSIONED if it is removed on the master, and
-LOST if it has not been seen by the master for a "long time".
-</dd><dt><span class="term">
-<code class="literal">getconfigs</code> 
-</span></dt><dd>
-This gets and dumps the configuration specifications of all
-the logical nodes the master knows about.
-</dd><dt><span class="term">
-<code class="literal">getmappings [<span class="emphasis"><em>physical node</em></span>]</code> 
-</span></dt><dd>
-Display all logical nodes mapped to
-<span class="emphasis"><em>physical node</em></span> or all mappings if <span class="emphasis"><em>physical node</em></span> is omitted.
-</dd><dt><span class="term">
-<code class="literal">exec</code> 
-</span></dt><dd>
-Synchronously execute a command on the master.  This command will
-block until it is completed.
-</dd><dt><span class="term">
-<code class="literal">source <span class="emphasis"><em>file</em></span></code> 
-</span></dt><dd>
-Reads the specified file and attempts to execute all of the
-specified commands.
-</dd><dt><span class="term">
-<code class="literal">submit</code> 
-</span></dt><dd>
-Asynchronously execute a command on the master.  This command will
-return immediately and allows the submission of other commands.  The command
-ID of the last command submitted is recorded.
-</dd><dt><span class="term">
-<code class="literal">wait <span class="emphasis"><em>ms</em></span> [<span class="emphasis"><em>cmdid</em></span>]</code> 
-</span></dt><dd>
-This commands blocks for up to <code class="literal">ms</code> milliseconds
-until <code class="literal">cmdid</code> has entered the SUCCEEDED or FAILED state. If <code class="literal">ms</code> is 0 the
-command may block forever.  If the command times out, the shell will
-disconnect.  This is useful in conjunction with <code class="literal">submitted</code> commands.
-</dd><dt><span class="term">
-<code class="literal">waitForNodesActive <span class="emphasis"><em>ms</em></span> node1 [node2 [&#8230;]]</code> 
-</span></dt><dd>
-This command blocks for up to
-<code class="literal">ms</code> milliseconds until the specified list of nodes have entered the ACTIVE or
-CONFIGURING state.  If ms==0 then the command may block forever.
-</dd><dt><span class="term">
-<code class="literal">waitForNodesDone <span class="emphasis"><em>ms</em></span> node1 [node2 [&#8230;]]</code> 
-</span></dt><dd>
-This command blocks for up to
-<code class="literal">ms</code> milliseconds until the specified list of nodes have entered the IDLE,
-ERROR, or LOST state.
-</dd><dt><span class="term">
-<code class="literal">quit</code> 
-</span></dt><dd>
-Exit the shell.
-</dd></dl></div><p title="Exec and Submit commands"><b>Exec and Submit commands.�</b>Both the web form and the FlumeShell are interfaces to the same command
-processing infrastructure inside Flume. This section introduces the
-FlumeShell and show how you can use it to make administering Flume more
-simple.</p><p>These commands are issued and run as if run from the master.  In the command
-shell they have the form:</p><p><code class="literal">exec <span class="emphasis"><em>command</em></span> [<span class="emphasis"><em>arg1 [_arg2</em></span> [ &#8230; ] ] ]</code></p><p><code class="literal">submit <span class="emphasis"><em>command</em></span> [<span class="emphasis"><em>arg1 [_arg2</em></span> [ &#8230; ] ] ]</code></p><p>Complex arguments like those with spaces, or non alpha-numeric characters can
-be expressed by using "double quotes"s and `single quotes&#8217;s.  If enclosed in
-double quotes, the bodies of the strings are Java string unescaped.  If they
-are enclosed in single quotes, arbitrary characters can be included except for
-the ' character.</p><p><code class="literal">exec</code> commands block until they are completed.  <code class="literal">submit</code> commands are
-asynchronously sent to the master in order to be executed.  <code class="literal">wait</code> are
-essentially joins for recently <code class="literal">submit</code> ted commands.</p><div class="variablelist"><dl><dt><span class="term">
-<code class="literal">noop</code> 
-</span></dt><dd>
-This command contacts the master and issues a noop (no
-operation) command.
-</dd><dt><span class="term">
-<code class="literal">config <span class="emphasis"><em>logicalnode</em></span> <span class="emphasis"><em>source</em></span> <span class="emphasis"><em>sink</em></span></code> 
-</span></dt><dd>
-This command configures a node.
-This is nearly identical to the <span class="emphasis"><em>config</em></span> command.
-</dd><dt><span class="term">
-<code class="literal">multiconfig <span class="emphasis"><em><span class="emphasis"><em>flumespec</em></span></em></span></code> 
-</span></dt><dd>
-This command configures a set of nodes on the
-master using the aggregated format.
-</dd><dt><span class="term">
-<code class="literal">unconfig <span class="emphasis"><em>logicalnode</em></span></code> 
-</span></dt><dd>
-This command changes the configuration of a
-particular node to have a <code class="literal">null</code> source and a <code class="literal">null</code> state.  +refresh
-</dd><dt><span class="term">
-<span class="emphasis"><em>logicalnode</em></span>+ 
-</span></dt><dd>
-This command refreshes the current configuration of a
-logical node.  This forces the logicalnode to stop and then restart.  This
-also causes a master re-evaluation that may change the failover lists.
-</dd><dt><span class="term">
-<code class="literal">refreshAll <span class="emphasis"><em>logicalnode</em></span></code> 
-</span></dt><dd>
-This atomically issues a refresh command to all
-of the logical nodes.
-</dd><dt><span class="term">
-<code class="literal">save <span class="emphasis"><em><span class="emphasis"><em>filename</em></span></em></span></code> 
-</span></dt><dd>
-This saves the current configuration to the master&#8217;s
-disk.
-</dd><dt><span class="term">
-<code class="literal">load <span class="emphasis"><em><span class="emphasis"><em>filename</em></span></em></span></code> 
-</span></dt><dd>
-This augments the current configuration with the
-logical node specifications found in <code class="literal"><span class="emphasis"><em>filename</em></span></code>.
-</dd><dt><span class="term">
-<code class="literal">map <span class="emphasis"><em>physicalnode</em></span> <span class="emphasis"><em>logicalnode</em></span></code>
-</span></dt><dd>
-This creates a new mapping between logical
-node <span class="emphasis"><em>logicalnode</em></span> and physical node <code class="literal"><span class="emphasis"><em>physicalnode</em></span></code>.  The node starts with a
-<code class="literal">null</code> source and a <code class="literal">null</code> sink, and updates its configuration specified at the
-master when it begins heartbeating.  Thus if a logical node configuration
-already exists and is mapped, it will pick up the configuration for the logical
-node.
-</dd><dt><span class="term">
-<code class="literal">spawn <span class="emphasis"><em>physicalnode</em></span> <span class="emphasis"><em>logicalnode</em></span></code>
-</span></dt><dd>
-The <code class="literal">spawn: command is a synonym for the
-+map</code> command and has been deprecated.
-</dd><dt><span class="term">
-<code class="literal">decommission <span class="emphasis"><em>logicalnode</em></span></code> 
-</span></dt><dd>
-This removes a logical node from the logical
-node configuration table, and unmaps it from any physical nodes it may be
-installed on.
-</dd><dt><span class="term">
-<code class="literal">unmap <span class="emphasis"><em>physicalnode</em></span> <span class="emphasis"><em>logicalnode</em></span></code> 
-</span></dt><dd>
-This command breaks the assignment of
-a <span class="emphasis"><em>logicalnode</em></span> from machine <span class="emphasis"><em>physicalnode</em></span>.  A logical node can be reassigned
-to another physical node using the <code class="literal">map</code> command.
-</dd><dt><span class="term">
-<code class="literal">unmapAll</code> 
-</span></dt><dd>
-This command breaks the assignment of all logical node from
-physical nodes.  A logical node can be reassigned to another physical node
-using the <code class="literal">map</code> command.
-</dd></dl></div></div></div><div class="section" title="8.2.�Flume&#8217;s Dataflow Specification Language"><div class="titlepage"><div><div><h3 class="title"><a name="_flume_8217_s_dataflow_specification_language"></a>8.2.�Flume&#8217;s Dataflow Specification Language</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_special_sinks_fan_out_fail_over_and_roll">8.2.1. Special Sinks: Fan out, Fail over, and Roll</a></span></dt><dt><span class="section"><a href="#_introducing_sink_decorators">8.2.2. Introducing Sink Decorators</a></span></dt><dt><span class="section"><a href="#_translations_of_high_level_sources_and_sinks">8.2.3. Translations of High-level Sources and Sinks</a></span></dt></dl></div><p>Using the Flume node roles (collector, agent) is the simplest method to get up
-and running with Flume.  Under the covers, these sources and sinks
-(collectorSink, collectorSource, and agentSource) are actually composed of
-primitive sinks that have been augmented with <span class="strong"><strong>translated components</strong></span> with
-role defaults, <span class="strong"><strong>special sinks</strong></span> and <span class="strong"><strong>sink decorators</strong></span>. These components make
-configuration more flexible, but also make configuration more complicated.
-The combination of special sinks and decorators expose a lot of details of the
-underlying mechanisms but are a powerful and expressive way to encode rich
-behavior.</p><p>Flume enables users to enter their own composition of sinks, sources, and
-decorators by using a domain specific data flow language.  The following
-sections will describe this in more details.</p><pre class="screen">nodeName      ::=  NodeId
-simpleSource  ::=  SourceId args?
-simpleSink    ::=  SinkId args?
-decoratorSink ::=  DecoId args?
-
-source ::= simpleSource
-
-sink ::=   simpleSink            // single sink
-     |     [ sink (, sink)* ]    // fanout sink
-     |     { decoratorSink =&gt; sink }  // decorator sink
-     |     &lt; sink ? sink &gt;           // failover / choice sink
-     |     roll(...) { sink }           // roll sink
-     |     let SinkId := sink in sink  // let expression
-
-logicalNode ::= NodeId : source | sink ;
-
-spec   ::=  (logicalNode)*</pre><div class="section" title="8.2.1.�Special Sinks: Fan out, Fail over, and Roll"><div class="titlepage"><div><div><h4 class="title"><a name="_special_sinks_fan_out_fail_over_and_roll"></a>8.2.1.�Special Sinks: Fan out, Fail over, and Roll</h4></div></div></div><p>Three special sinks are FanOutSinks, FailoverSinks, and RollSinks.</p><p>Fanout sinks send any incoming events to all of the sinks specified to be its
-children.  These can be used for data replication or for processing data off
-of the main reliable data flow path.  Fanout is similar to the Unix <code class="literal">tee</code>
-command or logically acts like an AND operator where the event is sent to each
-subsink.</p><p>The syntax for a FanoutSink is :</p><pre class="screen">[ console, collectorSink ]</pre><p>FailoverSinks are used to handle failures when appending new events.
-FailoverSinks can be used to specify alternate collectors to contact in the
-event the primary collector fails, or a local disk sink to store data until
-the primary collector recovers.  Failover is similar to exception handling and
-logically acts like an OR operator. If a failover is successful, one of the
-subsinks has received the event.</p><p>The syntax for a FailoverSink is :</p><pre class="screen">&lt; logicalSink("collector1") ? logicalSink("collector2") &gt;</pre><p>So, you could configure node "agent1" to have a failover to collector2 if
-collector1 fails (for example, if the connection to <code class="literal">collector1</code> goes down or
-if <code class="literal">collector1</code>'s HDFS becomes full):</p><pre class="screen">agent1 : source | &lt; logicalSink("collector1") ? logicalSink("collector2")
-&gt; ;</pre><p>Roll sink opens and closes a new instance of its subsink every
-<span class="emphasis"><em>millis</em></span> milliseconds.  A roll is an atomic transition that closes
-the current instance of the sub-sink, and then opens a new instance of
-as an escape sequence to differentiate data in different roll periods.
-This can be used to make every roll produce a new file with a unique
-name.</p><p>The syntax for a roll sink is:</p><pre class="screen">roll(millis) sink</pre><p>These can be composed to have even richer behavior.  For example, this sink
-outputs to the console and has a failover collector node.</p><pre class="screen">[ console, &lt; logicalSink("collector1") ? logicalSink("collector2") &gt; ]</pre><p>This one rolls the collector every 1000 milliseconds writing to a different
-HDFS file after each roll.</p><pre class="screen">roll(1000) [ console, escapedCustomDfs("hdfs://namenode/flume/file-%{rolltag}") ]</pre></div><div class="section" title="8.2.2.�Introducing Sink Decorators"><div class="titlepage"><div><div><h4 class="title"><a name="_introducing_sink_decorators"></a>8.2.2.�Introducing Sink Decorators</h4></div></div></div><p>Fan out and failover affect where messages go in the system but do not modify
-the messages themselves.  To augment or filter events as they pass through the
-dataflow, you can use <span class="strong"><strong>sink decorators</strong></span>.</p><p>Sink decorators can add properties to the sink and can modify the data streams
-that pass through them.  For example, you can use them to increase reliability
-via write ahead logging, increase network throughput via batching/compression,
-sampling, benchmarking, and even lightweight analytics.</p><p>The following simple sampling example uses an intervalSampler which is
-configured to send every 10th element from source "source" to the sink "sink":</p><pre class="screen">flumenode: source | { intervalSampler(10) =&gt; sink };</pre><p>Here&#8217;s an example that batches every 100 events together.</p><pre class="screen">flumenode: source | { batch(100) =&gt; sink };</pre><p>Like fanout and failover, decorators are also composable. Here is an example
-that creates batches of 100 events and then compresses them before moving
-them off to the sink:</p><pre class="screen">flumenode: source | { batch(100) =&gt;  { gzip =&gt; sink } };</pre></div><div class="section" title="8.2.3.�Translations of High-level Sources and Sinks"><div class="titlepage"><div><div><h4 class="title"><a name="_translations_of_high_level_sources_and_sinks"></a>8.2.3.�Translations of High-level Sources and Sinks</h4></div></div></div><p>Internally, Flume translates sinks into compositions of other simpler
-decorators, failovers, rollers to add properties.  The proper compositions
-create pipelines that provide different levels of reliability in Flume.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>This section describes how agents work, but this version of
-Flume does not expose the translations the same way the auto*Chains
-are exposed.  A future version of Flume will expose these details.
-The exact translations are still under development.</p></td></tr></table></div><p>Suppose you have nodes using the different agent sinks:</p><pre class="screen">node1 : tail("foo") | agentE2ESink("bar");
-node2 : tail("foo") | agentDFOSink("bar");
-node3 : tail("foo") | agentBESink("bar");</pre><p>In the translation phases, agentE2ESink is actually converted into these Flume
-sinks:</p><pre class="screen">node1 : tail("foo") | { ackedWriteAhead =&gt; { lazyOpen =&gt; { stubbornAppend
-=&gt; logicalSink("bar") } } } ;
-node2 : tail("foo") | let primary := { lazyOpen -&gt; {stubbornAppend =&gt; logicalSink("bar") } } in &lt; primary ? { diskFailover =&gt; { insistentOpen =&gt; primary} } &gt;;
-node3 : tail("foo") | { lazyOpen =&gt; { stubbornAppend =&gt;  logicalSink("bar")  } };</pre><p><code class="literal">ackedWriteAhead</code> is actually a complicated decorator that internally
-uses rolls and some other special decorators.  This decorator
-interface allows us to manually specify wire batching and compression
-options.  For example, you could compress every 100 messages using
-gzip compression:</p><pre class="screen">node1 : tail("foo") | { ackedWriteAhead =&gt; { batch(100) =&gt; { gzip =&gt; {
-lazyOpen =&gt; { stubbornAppend =&gt; logicalSink("bar") } } } } };</pre><p><code class="literal">collectorSink("xxx","yyy",15000)</code> is also a bit complicated with some custom
-decorators to handle acks.  Under the covers however, it depends on a roller
-with a escapedCustomDfsSink inside of it.</p><pre class="screen">roll(15000) { collectorMagic =&gt;  escapedCustomDfs("xxx", "yyy-%rolltag") }</pre><p>Another place translations happen is with logical nodes.  Lets start of with a
-few nodes:</p><pre class="screen">node1 : tail("foo") | { .... =&gt; logicalSink("node2") };
-node2 : logicalSource | collectorSink("...");</pre><p>The translation mechanisms converts logicalSources and logicalSinks into lower
-level physical rpcSources and rpcSinks.  Lets assume that node1 is on machine
-host1 and node2 is on machine host2.  After the translation you end up with the
-following configurations:</p><pre class="screen">node1 : tail("foo") | { .... =&gt; rpcSink("host2",12345) };
-node2 : rpcSource(12345) | collectorSink("..."");</pre><p>Suppose that you swap the mapping so that node2 is now on host1 and node1 is
-on host2.</p><pre class="screen"># flume shell commands
-exec unmapAll
-exec map host1 node2
-exec map host2 node1</pre><p>The original configuration will now be translated to:</p><pre class="screen">node1 : tail("foo") | { .... =&gt; rpcSink("host1",12345) };
-node2 : rpcSource(12345) | collectorSink("..."");</pre></div></div><div class="section" title="8.3.�Custom Metadata Extraction"><div class="titlepage"><div><div><h3 class="title"><a name="_custom_metadata_extraction"></a>8.3.�Custom Metadata Extraction</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_extractors">8.3.1. Extractors</a></span></dt><dt><span class="section"><a href="#_meta_data_filtering_and_transformations">8.3.2. Meta Data Filtering and Transformations</a></span></dt><dt><span class="section"><a href="#_role_defaults">8.3.3. Role Defaults</a></span></dt><dt><span class="section"><a href="#_arbitrary_data_flows_and_custom_architectures">8.3.4. Arbitrary Data Flows and Custom Architectures</a></span></dt></dl></div><p>While Flume can take in raw data, you can add structure to data based on the
-nodes they flowed through, and filter out portions of data to minimize the
-amount of raw data needed if only a portion is needed.</p><p>The simplest is the <code class="literal">value</code> decorator.  This takes two arguments: an attribute
-name, and a value to add to the event&#8217;s metadata.  This can be used to
-annotate constant information about the source, or arbitrary data to a
-particular node.  It could also be used to naively track the provenance of
-data through Flume.</p><div class="section" title="8.3.1.�Extractors"><div class="titlepage"><div><div><h4 class="title"><a name="_extractors"></a>8.3.1.�Extractors</h4></div></div></div><p>Extraction operations are also available to extract values from logs that
-have known structure.</p><p>One example is the <code class="literal">regex</code> decorator.  This takes three arguments: a regular
-expression, an index, and an attribute name which allows the user to use a
-extract a particular regex group out of the body of the event and write it as
-the value of the specified attribute.</p><p>Similarly the <code class="literal">split</code> decorator also takes three arguments: a regular
-expression, an index, and an attribute name.  This splits the body
-based on the regular expression, extracts the text group after the
-instance of the separator, and writes the value to the specified
-attribute.  One can think of this as a much simplified version of the
-<code class="literal">awk</code> Unix utility.</p></div><div class="section" title="8.3.2.�Meta Data Filtering and Transformations"><div class="titlepage"><div><div><h4 class="title"><a name="_meta_data_filtering_and_transformations"></a>8.3.2.�Meta Data Filtering and Transformations</h4></div></div></div><p>Flume enforces an invariant that prevents the modification of an attribute
-that has already been written upstream.  This simplifies debugging of
-dataflows, and improves the visibility of data when debugging.</p><p>If many stages are used, however, frequently a lot of extra metadata ends up
-being moved around.  To deal with this, a few extra operations are available.</p><p>A <code class="literal">select</code> operation is available.  This operation is like SQL select, which
-provides a relational calculus&#8217;s set projection operation that modifies an
-event so that the specified metadata fields are forwarded.</p><p>A <code class="literal">mask</code> operation is also available that forwards all metadata attributes
-except for the attributes specified.</p><p>A <code class="literal">format</code> decorator is also available that uses the escape mechanisms
-to rewrite the body of an event to a user customizable message.  This is
-useful for outputting summary data to low volume sources.  Example: writing
-summary information out to an IRC channel periodically.</p></div><div class="section" title="8.3.3.�Role Defaults"><div class="titlepage"><div><div><h4 class="title"><a name="_role_defaults"></a>8.3.3.�Role Defaults</h4></div></div></div><p>To simplify things for users, you can assign a particular role to a logical
-node&#8201;&#8212;&#8201;think of these as an "automatic" specification that have
-default settings.  Two roles currently provided are the agent role and the
-collector role.  Since there are many nodes with the same role, each of
-these are called a tier.  So, for example, the agent tier consists of all the nodes
-that are in the agent role.  Nodes that have the collector role are in the
-collector tier.</p><p>Agents and collectors roles have defaults specified in the <code class="literal">conf/flume-site.xml</code>
-file.  Look at the <code class="literal">conf/flume-conf.xml</code> file for properties prefixed with
-<code class="literal">flume.agent.*</code> and <code class="literal">flume.collector.*</code> for descriptions of the configuration
-options.</p><p>Each node maintains it own node state and has its own configuration. If the
-master does not have a data flow configuration for the logical node, the
-logical node will remain in IDLE state.  If a configuration is present, the
-logical node will attempt to instantiate the data flow and have it work
-concurrently with other data flows.</p><p>This means that each machine essentially only lives in one tier.  In a more
-complicated setup, it is possible to have a machine that contains many logical
-nodes, and because each of these nodes can take on different roles, the
-machine lives in multiple tiers.</p></div><div class="section" title="8.3.4.�Arbitrary Data Flows and Custom Architectures"><div class="titlepage"><div><div><h4 class="title"><a name="_arbitrary_data_flows_and_custom_architectures"></a>8.3.4.�Arbitrary Data Flows and Custom Architectures</h4></div></div></div><p>With tsinks and tsource, data can be sent through multiple nodes.  If ack
-injection and ack checking decorators are properly inserted, you can achieve
-reliability.</p></div></div><div class="section" title="8.4.�Extending via Sink/Source/Decorator Plugins"><div class="titlepage"><div><div><h3 class="title"><a name="_extending_via_sink_source_decorator_plugins"></a>8.4.�Extending via Sink/Source/Decorator Plugins</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_semantics_of_flume_extensions">8.4.1. Semantics of Flume Extensions</a></span></dt><dd><dl><dt><span class="section"><a href="#_simple_source_semantics">8.4.1.1. Simple source semantics.</a></span></dt><dt><span class="section"><a href="#_buffered_source_semantics">8.4.1.2. Buffered source semantics</a></span></dt><dt><span class="section"><a href="#_simple_sinks">8.4.1.3. Simple Sinks.</a></span></dt><dt><span class="section"><a href="#_buffered_sink_and_decorator_semantics">8.4.1.4. Buffered sink and decorator semantics</a></span></dt><dt><span class="section"><a href="#_retries_sleeps_and_unclean_exits">8.4.1.5. Retries, sleeps, and unclean exits.</a></span></dt></dl></dd></dl></div><p>An experimental plugin mechanism is provided that allows you to add new custom
-sources, sinks, and decorators to the system.</p><div class="orderedlist" title="Two steps are required to use this feature."><p class="title"><b>Two steps are required to use this feature.</b></p><ol class="orderedlist" type="1"><li class="listitem">
-First, add the jar with the new plugin classes to flume&#8217;s classpath.
-  If the plugin requires DLL&#8217;s/so&#8217;s make sure these are in the
-  LD_LIBRARY_PATH (unix .so) or PATH (windows .dll)
-</li><li class="listitem">
-Second, in <code class="literal">flume-site.xml</code>, add the class names of the new sources,
-  sinks, and/or decorators to the <code class="literal">flume.plugin.classes</code> property.
-  Multiple classes can be specified by comma separating the list. Java
-  reflection is used to find some special static methods that add new
-  components to the system and data flow language&#8217;s library.
-</li></ol></div><p>An example component has been "pluginified"&#8201;&#8212;&#8201;the "HelloWorld"
-source, sink, and decorator. This plugin does something very simple;
-the source generates the text "hello world!" every three seconds, the
-sink writes events to a "helloworld.txt" text file, and the decorator
-prepends "hello world!" to any event it encounters.</p><div class="orderedlist"><ol class="orderedlist" type="1"><li class="listitem">
-cd into the <code class="literal">plugins/helloworld</code> directory and type <code class="literal">ant</code>, a <code class="literal">helloworld_plugin.jar</code> file will be generated
-</li><li class="listitem"><p class="simpara">
-Add the following to flume-site.xml (create it if it doesn&#8217;t already
-  exist)
-  "helloworld.HelloWorldSink,helloworld.HelloWorldSource,helloworld.HelloWorldDecorator"
-  to the <code class="literal">flume.plugin.classes</code> property in <code class="literal">flume-site.xml</code>.
-</p><div class="important" title="Important" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Important"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Important]" src="images/important.png"></td><th align="left">Important</th></tr><tr><td align="left" valign="top"><p>if you use the provided <code class="literal">flume-site.xml.template</code> file to
-create your <code class="literal">flume-site.xml</code> be sure to comment out or remove any
-example properties contained in the sample template.</p></td></tr></table></div><p title="Example flume-site.xml contents"><b>Example flume-site.xml contents.�</b>
-</p><pre class="programlisting">&lt;configuration&gt;
-  &lt;property&gt;
-    &lt;name&gt;flume.plugin.classes&lt;/name&gt;
-    &lt;value&gt;helloworld.HelloWorldSink,helloworld.HelloWorldSource,helloworld.HelloWorldDecorator&lt;/value&gt;
-    &lt;description&gt;Comma separated list of plugins&lt;/description&gt;
-  &lt;/property&gt;
-&lt;/configuration&gt;</pre><p title="Example flume-site.xml contents">
-</p></li><li class="listitem"><p class="simpara">
-Start the Flume master and at least one logical node in separate terminals
-</p><div class="orderedlist"><ol class="orderedlist" type="a"><li class="listitem">
-in each terminal cd into the top-level flume directory, should be just above <code class="literal">plugins</code>
-</li><li class="listitem"><p class="simpara">
-Add <code class="literal">helloworld_plugin.jar</code> to the FLUME_CLASSPATH in <span class="strong"><strong>both</strong></span> terminals
-</p><pre class="screen">export FLUME_CLASSPATH=`pwd`/plugins/helloworld/helloworld_plugin.jar</pre></li><li class="listitem">
-in terminal 1 run <code class="literal">bin/flume master</code>
-</li><li class="listitem">
-in terminal 2 run <code class="literal">bin/flume node -n hello1</code>
-</li></ol></div></li><li class="listitem"><p class="simpara">
-At this point the master and hello1 nodes should be started and will have loaded the plugin
-</p><p title="You should see log output similar to the following in both master and hello1:"><b>You should see log output similar to the following in both master and hello1:�</b>
-</p><pre class="screen">10/07/29 17:35:28 INFO conf.SourceFactoryImpl: Found source builder helloWorldSource in helloworld.HelloWorldSource
-10/07/29 17:35:28 INFO conf.SinkFactoryImpl: Found sink builder helloWorldSink in helloworld.HelloWorldSink
-10/07/29 17:35:28 INFO conf.SinkFactoryImpl: Found sink decorator helloWorldDecorator in helloworld.HelloWorldDecorator</pre><p title="You should see log output similar to the following in both master and hello1:">
-</p><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>Another way to verify that your plugin is loaded is to check if it is displayed on this page <a class="ulink" href="http://localhost:35871/masterext.jsp" target="_top">http://localhost:35871/masterext.jsp</a></p></td></tr></table></div></li><li class="listitem"><p class="simpara">
-Configure hello1
-</p><div class="tip" title="Tip" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Tip"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Tip]" src="images/tip.png"></td><th align="left">Tip</th></tr><tr><td align="left" valign="top"><p>The easiest way to do this is open the configuration page of the master in a browser, typically this link <a class="ulink" href="http://localhost:35871/flumeconfig.jsp" target="_top">http://localhost:35871/flumeconfig.jsp</a></p></td></tr></table></div><div class="orderedlist"><ol class="orderedlist" type="a"><li class="listitem"><p class="simpara">
-load the helloworld source/sink into our <code class="literal">hello1</code> node (the bottom text box, then submit button if you are using the master&#8217;s web interface"
-</p><pre class="screen">hello1: helloWorldSource() | helloWorldSink();</pre></li><li class="listitem"><p class="simpara">
-you could also try the hello world decorator
-</p><pre class="screen">hello1: helloWorldSource() | { helloWorldDecorator() =&gt; helloWorldSink() };</pre><p class="simpara">In either case <code class="literal">hello1</code> will output a <code class="literal">helloworld.txt</code> file into it&#8217;s current working directory. Every 3 seconds a new "hello world!" line will be output to the file.</p></li></ol></div></li></ol></div><div class="section" title="8.4.1.�Semantics of Flume Extensions"><div class="titlepage"><div><div><h4 class="title"><a name="_semantics_of_flume_extensions"></a>8.4.1.�Semantics of Flume Extensions</h4></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_simple_source_semantics">8.4.1.1. Simple source semantics.</a></span></dt><dt><span class="section"><a href="#_buffered_source_semantics">8.4.1.2. Buffered source semantics</a></span></dt><dt><span class="section"><a href="#_simple_sinks">8.4.1.3. Simple Sinks.</a></span></dt><dt><span class="section"><a href="#_buffered_sink_and_decorator_semantics">8.4.1.4. Buffered sink and decorator semantics</a></span></dt><dt><span class="section"><a href="#_retries_sleeps_and_unclean_exits">8.4.1.5. Retries, sleeps, and unclean exits.</a></span></dt></dl></div><p>Flume uses threading to support reconfiguration and multiple logical
-nodes.  Sources, sinks, and decorator are extensions that can use
-queues for buffering and can use clock sleeps for periodic rolls and
-retries.  Because we use these blocking operations and because the
-internal concurrency control mechanisms we use can cause deadlocks or
-hangs, there are several rules that need to be followed and enforced
-by test cases in order to be a well-behaved source, sink, or
-decorator.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>This section is a draft and not exhaustive or completely formal.
-More restrictions may be added to this in the future.</p></td></tr></table></div><p title="Semantics of sources"><b>Semantics  of sources.�</b>Sources must implement 4 methods</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-void open() throws IOException
-</li><li class="listitem">
-Event next() throws IOException
-</li><li class="listitem">
-void close() throws IOException
-</li><li class="listitem">
-ReportEvent getReport()
-</li></ul></div><p>Along with these signatures, each of these methods can also throw
-RuntimeExceptions.  These exceptions indicate a failure condition and
-by default will make a logical node shutdown in ERROR state.  These
-error messages are user visible and it is important that they have
-actionable descriptions about the failure conditions without having to
-dump stack.  Thus, NullPointerExceptions are not acceptable&#8201;&#8212;&#8201;they
-are not descriptive or helpful without their stack traces.</p><p>Some examples of valid runtime exceptions include invalid arguments
-that prevent a source from opening and invalid state infractions
-(attempting to next on a closed source).</p><div class="section" title="8.4.1.1.�Simple source semantics."><div class="titlepage"><div><div><h5 class="title"><a name="_simple_source_semantics"></a>8.4.1.1.�Simple source semantics.</h5></div></div></div><div class="informalfigure"><div class="mediaobject"><img src="sourceStatesSimple.png" alt="sourceStatesSimple.png"></div></div><p>Simple sources are assumed to be sources whose open and close
-operation happen quickly and do not block for long periods of time.
-The longest pauses tolerated here on the order of 10s (default time
-for a failing DNS lookup).</p><p>The constructor for the sources should be callable without grabbing
-resources that can block or that require IO such as network connectors
-or file handles.  If there are errors due to configuration settings
-that can be caught in the constructor, an IllegalArgumentException
-should be thrown.</p><p><code class="literal">open()</code> is a call that grabs resources for the source so that the
-<code class="literal">next()</code> call can be made.  The <code class="literal">open</code> call of a simple source should
-attempt to fail fast. It can throw a IOException or a RuntimeException
-such as IllegalStateException.  Open should only be called on a CLOSED
-sink&#8201;&#8212;&#8201;if a sink is opened twice, the second call should throw an
-IOException or IllegalStateException.</p><p><code class="literal">next()</code> is a call that gets an event from an open source.  If a
-source is not open, this should throw an IllegalStateException.  This
-call can and will often block. If <code class="literal">next()</code> is blocking, a call to
-<code class="literal">close()</code> should unblock the <code class="literal">next()</code> by having it exit cleanly
-returning null.  Many resources such as TCP-sockets (and ideally rpc
-frameworks) default to throwing an exception on blocked network reads
-(like a <code class="literal">next()</code> call) when <code class="literal">close()</code>'d.</p><p><code class="literal">close()</code> is a call that releases resources that the open call of a
-source grabs.  The <code class="literal">close()</code> call itself blocks until all resources
-are released.  This allows a subsequent <code class="literal">open()</code> in the same thread to
-not fail due to resource contention (ex, closing a server socket on
-port 12345 should not return until port 12345 is ready to be bound
-again).</p><p><code class="literal">getReport()</code> returns a ReportEvent.  These values should be available
-regardless if the node is open or closed and this call should not
-block the other source by other calls (due to potential lock inversion
-issues).  The values retrieved are ideally atomically grabbed but this
-is not required as long as no errors are caused by racy execution.</p><p>If a source is opened or closed multiple times, it is up to the source to
-determine if values are reset or persisted between open/close cycles.</p></div><div class="section" title="8.4.1.2.�Buffered source semantics"><div class="titlepage"><div><div><h5 class="title"><a name="_buffered_source_semantics"></a>8.4.1.2.�Buffered source semantics</h5></div></div></div><div class="informalfigure"><div class="mediaobject"><img src="sourceStatesBuffered.png" alt="sourceStatesBuffered.png"></div></div><p>Some sources have in memory queues or buffers stored persistently on
-disk.  The guiding principle here is that on <code class="literal">close()</code>, buffered
-sources should prevent new data from entering and attempt to flush the
-buffered data.  Also any subordinate threads should be released before
-close returns.  If no progress is being made on <code class="literal">close()</code>, for a given
-period of time (30s is the default currently) the controlling thread
-will for an Thread.interrupt() call.  The source should be able to
-handle InterruptedExceptions and percolate interrupted status up the
-returning call stack.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>In v0.9.1 and v0.9.2, interruptions when caught should be
-handled by re-flagging the Thread&#8217;s interrupted flag (a call to
-<code class="literal">Thread.currentThread().interrupt()</code>) and then throwing an
-IOException.  The API for extensions will likely change in the future
-to throw either an IOException or an InterruptedException.</p></td></tr></table></div><p><code class="literal">open()</code> no change.</p><p><code class="literal">close()</code> This call is usually made from a separate thread than the
-open() or next() calls.  Since this call should block until resources
-are freed, it should attempt to flush its buffers before returning.
-For example, so if a network source has some buffered data, the
-network connection should be closed to prevent new data from entering,
-and then the buffered data should be flushed.  The source should be
-able to handle InterruptedExceptions and percolate interrupted status
-up the returning call stack and indicate an error state.</p><p>A <code class="literal">next()</code> call while in CLOSING state should continue pulling values
-out of the buffer until it is empty.  This is especially true if the
-next() call is happening in the main driver thread or a subordinate
-driver thread.  One mechanism to get this effect is to add a special
-DONE event to the queue/buffer that indicates a clean exit.</p><p><code class="literal">getReport()</code> ideally includes metric information like the size of the
-queues in the source, and the number of elements in the queue.</p><p title="Semantics of sinks and decorators"><b>Semantics of sinks and decorators.�</b>Sinks and decorators must implement 4 methods</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-void open() throws IOException
-</li><li class="listitem">
-void append(Event e) throws IOException
-</li><li class="listitem">
-void close() throws IOException
-</li><li class="listitem">
-ReportEvent getReport()
-</li></ul></div><p>Along with these signatures, each of these methods can also throw
-RuntimeExceptions.  Run-time exceptions indicate a failure condition
-and by default will make a logical node shutdown in ERROR state.
-These error messages are user visible and it is important that they
-have helpful descriptions about the failure conditions without having
-to dump stack.  Thus, NullPointerExceptions are not acceptable&#8201;&#8212;&#8201;they
-are not descriptive or helpful without their stack traces.</p></div><div class="section" title="8.4.1.3.�Simple Sinks."><div class="titlepage"><div><div><h5 class="title"><a name="_simple_sinks"></a>8.4.1.3.�Simple Sinks.</h5></div></div></div><div class="informalfigure"><div class="mediaobject"><img src="sinkStatesSimple.png" alt="sinkStatesSimple.png"></div></div><p>Simple sinks are assumed to have open, close, and append operations
-that happen quickly and do not block for long periods of time.  The longest
-pauses tolerated here on the order of 10s (default time for a failing
-DNS lookup).</p><p>The constructor for the sinks and decorators should be callable
-without grabbing resources that can block or that require IO such as
-network connectors or file handles.  If there are errors due to
-configuration settings that can be caught in the constructor, an
-IllegalArgumentException should be thrown.</p><p><code class="literal">open()</code> is a call that grabs resources for the sink or decorator so
-that the <code class="literal">append(Event)</code> call can be made.  If there are errors due to
-configuration settings not detectable without IO in the constructor,
-<code class="literal">open()</code> should attempt to fail fast and throw a IOException or
-RuntimeException such as IllegalStateException or
-IllegalArgumentException.  Open should only be called on a CLOSED sink&#8201;&#8212;&#8201;if a sink is opened twice, the second call should throw an
-IOException or IllegalStateException.</p><p><code class="literal">append()</code> is a call that delivers a event.  If a sink is not open,
-this should throw an IllegalStateException.</p><p>If a normal decorator fails to open or to append because of an
-internal failure or a subsink fails to open, the decorator should
-release its resources attempt to close the subsink and then throw an
-exception.  There are some sink/decos that specifically manipulate
-these semantics&#8201;&#8212;&#8201;this needs to be done with care.</p><p><code class="literal">close()</code> is a call that releases resources that the open call of a
-source grabs.  If <code class="literal">open()</code> or <code class="literal">next()</code> is blocking, a call to this
-should unblock the call and have them exit.  <code class="literal">close()</code> should be
-called called on an open sink, but we allow a closed sink to have
-<code class="literal">close()</code> called on it without throwing an exception (generally LOG
-warning however).</p><p><code class="literal">getReport()</code> returns a ReportEvent.  These values should be available
-regardless if the node is open or closed and this call should not
-cause get blocked by other calls (due to potential lock inversion
-issues).  The values retrieved are ideally atomically grabbed but this
-is not required as long as no errors are caused by racy execution.  If
-a sink is opened or closed multiple times, it is up to the sink to
-determine if values are reset or persisted between open/close cycles.</p></div><div class="section" title="8.4.1.4.�Buffered sink and decorator semantics"><div class="titlepage"><div><div><h5 class="title"><a name="_buffered_sink_and_decorator_semantics"></a>8.4.1.4.�Buffered sink and decorator semantics</h5></div></div></div><div class="informalfigure"><div class="mediaobject"><img src="sinkStatesBuffered.png" alt="sinkStatesBuffered.png"></div></div><p>Some sinks have queues or buffers stored in memory or persistently on
-disk.  The guiding principle here is that buffered sinks should
-attempt to flush its buffers when prompted to <code class="literal">close()</code>.  This needs
-to be balanced with the requirement that sinks and decorated sinks
-should attempt to close in a relatively quick fashion.</p><p><code class="literal">open()</code> Since this sink isn&#8217;t open this generally means there is no
-buffered data.  However, an <code class="literal">open()</code> on a sink or decorator with
-persistent data should attempt to recover data and enqueue it in the
-<code class="literal">open()</code> call. Examples of these include a DFO or WAL log recovering
-dfo/wla logs, when a network subsink is down.</p><p>An <code class="literal">append()</code> call may buffer data before sending it (such as a
-batching decorator).  A <code class="literal">close()</code> call, should attempt to append
-buffered data to its (sub)sink before executing the close. Also, any
-subordinate threads should be stopped before shutdown.</p><p>If no progress is being made on <code class="literal">close()</code>, for a given period of time
-(30s is the default currently) it will be interrupted and should
-handle abruptly exiting because of this interruption.</p></div><div class="section" title="8.4.1.5.�Retries, sleeps, and unclean exits."><div class="titlepage"><div><div><h5 class="title"><a name="_retries_sleeps_and_unclean_exits"></a>8.4.1.5.�Retries, sleeps, and unclean exits.</h5></div></div></div><div class="informalfigure"><div class="mediaobject"><img src="sinkStatesOpening.png" alt="sinkStatesOpening.png"></div></div><p>Retry on open semantics</p><div class="informalfigure"><div class="mediaobject"><img src="sinkStatesAppending.png" alt="sinkStatesAppending.png"></div></div><p>Retry on Append semantics</p><p>Some decorators introduce retries and sleeps. An author who uses these
-needs to ensure that these operations behave well on open, append, and
-close.  When combined with buffering sinks, flushing a buffer on close
-may not be possible! (ex: wal trying to send data to a dead network
-connection).  This means these sinks/decos need a way to exit abruptly
-and report an error.  There are two operations that make these cases
- need to be handled: unbounded retries and unbounded/long
-sleeps/await/blocking.</p><p>Some decorators have potentially unbounded retry semantics.  For
-example, InsistentOpen, InsistentAppend, and FailoverSinks have the
-potential to retry <code class="literal">open()</code> and <code class="literal">append()</code> calls an unbounded number
-of times.  These decorators can also be wrapped with in others&#8201;&#8212;&#8201;this
-means we need to be able to percolate the hard exit and bypass the
-retry logic.</p><p>To do this we require that any sink/deco that has retry logic must
-check for hard exit before retrying.  These sinks must to propagate
-the hard exit interruption to its upstream decorators (in case they
-have retry logic!).</p><p>Some sinks have the potential to backoff or sleep for long or
-potentially unbounded amounts of time.  Code using sleeps or
-synchronization operations such as waiting on latches
-(<code class="literal">CountDownLatch.await()</code>) or thread sleeps (<code class="literal">Thread.sleep()</code>) must
-properly handle interruptions.  Since these yielding operations are
-usually only used in retry operations (which meant there was a
-failure), the sink/deco needs to propagate the interruption and fail
-with error.</p><p>There are some ramifications of these semantics.  Care must be taken
-with locking open, close, and append operations.  If there are any
-sleeps or blocking <code class="literal">open()</code> operations (ex: InsistentOpen,
-FailoverSinks), ideally a close call will cause it to shutdown, it and
-the open call should get unblocked.  <code class="literal">append()</code></p><p>The sink signaled to be closed but blocked on <code class="literal">append()</code> or <code class="literal">open()</code>
-should exit in a reasonable amount of time&#8201;&#8212;&#8201;ideally within a few
-heartbeats (5s is the default, so ideally &lt;30s).  If the sink exits
-and its buffers are empty, it should do a normal successful return.
-If there were unflushed events, it should return error by throwing an
-exception.  If there are subordinate threads, these should be
-terminated before close returns.</p></div></div></div><div class="section" title="8.5.�Limiting Data Transfer Rate between Source-Sink pairs"><div class="titlepage"><div><div><h3 class="title"><a name="_limiting_data_transfer_rate_between_source_sink_pairs"></a>8.5.�Limiting Data Transfer Rate between Source-Sink pairs</h3></div></div></div><p>Flume has the ability to limit the rate of transfer of data between source-sink pairs.
-This is useful to divide the network bandwidth non-uniformly among different
-source-sink pairs based on the kind of logs they are transferring.
-For example, you can ship some types of logs at a faster rate than others, or you can transfer logs at different rates at different times of the day.</p><p>Another example of when it is beneficial to limit the rate of data transfer is when the network (or a collector) recovers after a failure.
-In this case, the agents might have a lot of data backed up to ship; if there is no limit on the transfer rate,
-the agents can exhaust all of the resources of the collector and possibly cause it to crash.</p><p>In Flume, you can use a special sink-decorator called the <code class="literal">choke</code> decorator to limit the rate of transfer of data between source-sink pairs.
-Each <code class="literal">choke</code> decorator (called a <code class="literal">choke</code>) has to be assigned to a <code class="literal">choke-id</code>.
-Here is an example of using a <code class="literal">choke</code> between a source and sink of a node; the <code class="literal">choke-id</code> of this <code class="literal">choke</code> is "Cid".</p><pre class="screen">node: source | { choke("Cid") =&gt; sink };</pre><p>The <code class="literal">choke-ids</code> are specific to a physical node. Before using a <code class="literal">choke</code> on <code class="literal">node</code>,
-you must register the <code class="literal">choke-id</code> on the physical node containing the <code class="literal">node</code>.
-You can register a <code class="literal">choke-id</code> on a physical node using the <code class="literal">setChokeLimit</code> command.
-When registering a <code class="literal">choke-id</code>, you must also assign a rate limit (in KB/sec) to it.
-Here is an example of registering a <code class="literal">choke-id</code> "Cid" on a physical node <code class="literal">host</code> and assigning a limit 1000 KB/sec.</p><pre class="screen">exec setChokeLimit host Cid 1000</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>You can also use setChokeLimit at the run-time to change the limit assigned to a <code class="literal">choke-id</code>.</p></td></tr></table></div><p>The limit on the <code class="literal">choke-id</code> specifies an upper bound on the rate at which the <code class="literal">chokes</code> using that <code class="literal">choke-id</code> can collectively transfer data.
-In the preceding example, there is only one source-sink pair on the physical node <code class="literal">host</code> that uses a <code class="literal">choke</code> with <code class="literal">choke-id</code> "Cid". Consequently, the rate of data transfer between that source-sink pair is limited to 1000 KB/sec.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>For the purpose of rate limiting, only the size of the event body is taken into account.</p></td></tr></table></div><p>The <code class="literal">choke</code> decorator works as follows: when <code class="literal">append()</code> is called on the sink to which the <code class="literal">choke</code>
-is attached, the <code class="literal">append()</code> call works normally if the amount of data transferred
-(during a small duration of time) is within the limit assigned to the <code class="literal">choke-id</code> corresponding to the <code class="literal">choke</code>.
-If the limit has been exceeded, then <code class="literal">append()</code> is blocked for a small duration of time.</p><p>Suppose there are multiple source-sink pairs using <code class="literal">chokes</code> between them, and they are using
-the same <code class="literal">choke-id</code>.  Suppose further that both <code class="literal">node1</code> and <code class="literal">node2</code> are logical nodes on the same physical node <code class="literal">host</code>,
-and a <code class="literal">choke-id</code> "Cid" with limit 1000KB/sec is registered on <code class="literal">host</code>.</p><pre class="screen">node1: source1 | { choke("Cid") =&gt; sink1 };
-node2: source2 | { choke("Cid") =&gt; sink2 };</pre><p>In this example, because both <code class="literal">source1-sink1</code> and <code class="literal">source2-sink2</code> pairs are using <code class="literal">chokes</code> with the same <code class="literal">choke-id</code> "Cid", the total data going across these source-sink pairs collectively is limited to 1000KB/sec.
-Flume does not control how this limit will be divided between the source-sink pairs,
-but it does guarantee that neither source-sink pair will starve.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>If multiple source-sink pairs on the same physical node use chokes that have the same choke-id,
-then there is no guarantee how the rate limit will be divided between these source-sink pairs.</p></td></tr></table></div></div></div><div class="section" title="9.�Flume and HDFS Security Integration"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_flume_and_hdfs_security_integration"></a>9.�Flume and HDFS Security Integration</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_basics">9.1. Basics</a></span></dt><dt><span class="section"><a href="#_setting_up_flume_users_on_kerberos">9.2. Setting up Flume users on Kerberos</a></span></dt><dd><dl><dt><span class="section"><a href="#_administering_kerberos_principals">9.2.1. Administering Kerberos principals</a></span></dt></dl></dd></dl></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>This section is only required if you are using a Kerberized HDFS
-cluster.  If you are running CDH3b2 or a Hadoop version 0.21.x or
-earlier, you can safely skip this section.</p></td></tr></table></div><p>Flume&#8217;s datapath needs to be able to interact with "secured" Hadoop
-and HDFS.  The Hadoop and HDFS designers have chosen to use the
-Kerberos V5 system and protocols to authenticate communications
-between clients and services.  Hadoop clients include users, MR jobs
-on behalf of users, and services include HDFS, MapReduce.</p><p>In this section we will describe how setup up a Flume node to be a
-client as user <span class="emphasis"><em>flume</em></span> to a kerberized HDFS service.  This section
-will <span class="strong"><strong>not</strong></span> talk about securing the communications between Flume nodes
-and Flume masters, or the communications between Flume nodes in a
-Flume flow.  The current implementation does not support writing
-individual isolated flows as different users.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>This has only been tested with the security enhanced betas of
-CDH (CDH3b3+), and the MIT Kerberos 5 implementation.</p></td></tr></table></div><div class="section" title="9.1.�Basics"><div class="titlepage"><div><div><h3 class="title"><a name="_basics"></a>9.1.�Basics</h3></div></div></div><p>Flume will act as a particular Kerberos principal (user) and needs
-credentials.  The Kerberos credentials are needed in order to interact
-with the kerberized service.</p><p>There are two ways you can get credentials. The first is used by
-interactive users because it requires an interactive logon.  The
-second is generally used by services (like a Flume daemon) and uses a
-specially protected key table file called a <span class="emphasis"><em>keytab</em></span>.</p><p>Interactively using the <code class="literal">kinit</code> program to contact the Kerberos KDC
-(key distribution center) is one way is to prove your identity. This
-approach requires a user to enter a password.  To do this you need a
-two part principal setup in the KDC, which is generally of the form
-<code class="literal">user@REALM.COM</code>.  Logging in via <code class="literal">kinit</code> will grant a ticket granting
-ticket (TGT) which can be used to authenticate with other services.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>this user needs to have an account on the namenode machine as
-well&#8201;&#8212;&#8201;Hadoop uses this user and group information from that machine
-when authorizing access.</p></td></tr></table></div><p>Authenticating a user or a service can alternately be done using a
-specially protected <span class="emphasis"><em>keytab</em></span> file.  This file contains a ticket
-generating ticket (TGT) which is used to mutually authenticate the
-client and the service via the Kerberos KDC.</p><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>The keytab approach is similar to an "password-less" ssh
-connections.  In this case instead of an id_rsa private key file, the
-service has a keytab entry with its private key.</p></td></tr></table></div><p>Because a Flume node daemon is usually started unattended (via service
-script), it needs to login using the keytab approach.  When using a
-keytab, the Hadoop services requires a three part principal.  This has
-the form <code class="literal">user/host.com@REALM.COM</code>.  We recommend using <code class="literal">flume</code> as the
-user and the hostname of the machine as the service.  Assuming that
-Kerberos and kerberized Hadoop has been properly setup, you just need
-to a few parameters to the Flume node&#8217;s property file
-(flume-site.xml).</p><pre class="screen">&lt;property&gt;
-&lt;name&gt;flume.kerberos.user&lt;/name&gt;
-&lt;value&gt;flume/host1.com@REALM.COM &lt;/value&gt;
-&lt;description&gt;&lt;/description&gt;
-&lt;/property&gt;
-
-&lt;property&gt;
-&lt;name&gt;flume.kerberos.keytab&lt;/name&gt;
-&lt;value&gt;/etc/flume/conf/keytab.krb5 &lt;/value&gt;
-&lt;description&gt;&lt;/description&gt;
-&lt;/property&gt;</pre><p>In this case, <code class="literal">flume</code> is the user, <code class="literal">host1.com</code> is the service, and
-<code class="literal">REALM.COM</code> is the Kerberos realm.  The <code class="literal">/etc/keytab.krb5</code> file contains
-the keys necessary for <code class="literal">flume/host1.com@REALM.COM</code> to authenticate
-with other services.</p><p>Flume and Hadoop provides a simple keyword (_HOST) that gets expanded
-to be the host name of the machine the service is running on.  This
-allows you to have one flume-site.xml file with the same
-flume.kerberos.user property on all of your machines.</p><pre class="screen">&lt;property&gt;
-&lt;name&gt;flume.kerberos.user&lt;/name&gt;
-&lt;value&gt;flume/_HOST@REALM.COM &lt;/value&gt;
-&lt;description&gt;&lt;/description&gt;
-&lt;/property&gt;</pre><p>You can test to see if your Flume node is properly setup by running
-the following command.</p><pre class="screen">flume node_nowatch -1 -n dump -c 'dump: console |  collectorSink("hdfs://kerb-nn/user/flume/%Y%m%D-%H/","testkerb");'</pre><p>This should write data entered at the console to a kerberized HDFS
-with a namenode named kerb-nn, into a <code class="literal">/user/flume/YYmmDD-HH/</code>
-directory.</p><p>If this fails, you many need to check to see if Flume&#8217;s Hadoop
-settings (in core-site.xml and hdfs-site.xml) are using Hadoop&#8217;s
-settings correctly.</p></div><div class="section" title="9.2.�Setting up Flume users on Kerberos"><div class="titlepage"><div><div><h3 class="title"><a name="_setting_up_flume_users_on_kerberos"></a>9.2.�Setting up Flume users on Kerberos</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_administering_kerberos_principals">9.2.1. Administering Kerberos principals</a></span></dt></dl></div><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>These instructions are for MIT Kerb5.</p></td></tr></table></div><p>There are several requirements to have a "properly setup" Kerberos
-HDFS + Flume.</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-Need to have a prinicipal for the Flume user on each machine.
-</li><li class="listitem">
-Need to have a keytab that has keys for each principal on each machine.
-</li></ul></div><p>Much of this setup can be done by using the <code class="literal">kadmin</code> program, and
-verified using the <code class="literal">kinit</code>, <code class="literal">kdestroy</code>, and <code class="literal">klist</code> programs.</p><div class="section" title="9.2.1.�Administering Kerberos principals"><div class="titlepage"><div><div><h4 class="title"><a name="_administering_kerberos_principals"></a>9.2.1.�Administering Kerberos principals</h4></div></div></div><p>First you need to have permissions to use the <code class="literal">kadmin</code> program and the
-ability to add to principals to the KDCs.</p><pre class="screen">$ kadmin -p &lt;adminuser&gt; -w &lt;password&gt;</pre><p>If you entered this correctly, it will drop you do the kadmin prompt</p><pre class="screen">kadmin:</pre><p>Here you can add a Flume principal to the KDC</p><pre class="screen">kadmin: addprinc flume
-WARNING: no policy specified for flume@REALM.COM; defaulting to no policy
-Enter password for principal "flume@REALM.COM":
-Re-enter password for principal "flume@REALM.COM":
-Principal "flume@REALM.COM" created.
-kadmin:</pre><p>You also need to add principals with hosts for each Flume node that
-will directly write to HDFS.  Since you will be exporting the key to a
-keytab file, you can use the -randkey option to generate a random key.</p><pre class="screen">kadmin: addprinc -randkey flume/host.com
-WARNING: no policy specified for flume/host.com@REALM.COM; defaulting to no policy
-Principal "flume/host.com@REALM.COM" created.
-kadmin:</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>Hadoop&#8217;s Kerberos implementation requires a three part principal
-name&#8201;&#8212;&#8201;user/host@REALM.COM.  As a user you usually only need the user
-name, <a class="ulink" href="mailto:user@REALM.COM" target="_top">user@REALM.COM</a>.</p></td></tr></table></div><p>You can verify that the user has been added by using the <code class="literal">kinit</code>
-program, and entering the password you selected.  Next you can verify
-that you have your Ticket Granting Ticket (TGT) loaded.</p><pre class="screen">$ kinit flume/host.com
-Password for flume/host.com@REALM.COM:
-$ klist
-Ticket cache: FILE:/tmp/krb5cc_1016
-Default principal: flume/host.com@REALM
-
-Valid starting     Expires            Service principal
-09/02/10 18:59:38  09/03/10 18:59:38  krbtgt/REALM.COM@REALM.COM
-
-
-Kerberos 4 ticket cache: /tmp/tkt1016
-klist: You have no tickets cached
-$</pre><p>You can ignore the Kerberos 4 info.  To "logout" you can use the
-<code class="literal">kdestroy</code> command, and then verify that credentials are gone by
-running <code class="literal">klist</code>.</p><pre class="screen">$ kdestroy
-$ klist
-klist: No credentials cache found (ticket cache FILE:/tmp/krb5cc_1016)
-
-
-Kerberos 4 ticket cache: /tmp/tkt1016
-klist: You have no tickets cached
-$</pre><p>Next to enable automatic logins, we can create a keytab file so that
-does not require manually entering a password.</p><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p>This keytab file contains secret credentials that should be
-protected so that only the proper user can read the file.  After
-created, it should be in 0400 mode (-r--------) and owned by the user
-running the Flume process.</p></td></tr></table></div><p>Then you can generate a keytab file (int this example called
-<code class="literal">flume.keytab</code>) and add a user <code class="literal">flume/host.com</code> to it.</p><pre class="screen">kadmin: ktadd -k flume.keytab flume/host.com</pre><div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Note"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td><th align="left">Note</th></tr><tr><td align="left" valign="top"><p>This will invalidate the ability for flume/host.com to manually
-login of the account.  You could however have a Flume user does not
-use a keytab and that could log in.</p></td></tr></table></div><div class="warning" title="Warning" style="margin-left: 0.5in; margin-right: 0.5in;"><table border="0" summary="Warning"><tr><td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td><th align="left">Warning</th></tr><tr><td align="left" valign="top"><p><code class="literal">ktadd</code> can add keytab entries for mulitple principals into a
-single file and allow for a single keytab file with many keys.  This
-however weakens the security stance and may make revoking credentials
-from misbehaving machines difficult.  Please consult with your
-security administrator when assessing this risk.</p></td></tr></table></div><p>You can verify the names and the version (KVNO) of the keys by running
-the following command.</p><pre class="screen">$ klist -Kk flume.keytab
-Keytab name: FILE:flume.keytab
-KVNO Principal
----- --------------------------------------------------------------------------
-   5 flume/host.com@REALM.COM (0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)
-   5 flume/host.com@REALM.COM (0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)
-   5 flume/host.com@REALM.COM (0xcccccccccccccccc)
-   5 flume/host.com@REALM.COM (0xdddddddddddddddd)</pre><p>You should see a few entries and your corresponding keys in hex after
-your principal names.</p><p>Finally, you can use <code class="literal">kinit</code> with the <code class="literal">flume@REALM.COM</code> principal to
-interactively do a Kerberos login and use the Hadoop commands to browse HDFS.</p><pre class="screen">$ kinit flume
-Password for flume@REALM.COM:  &lt;-- enter password
-$ hadoop dfs -ls /user/flume/</pre></div></div></div><div class="section" title="10.�Appendix"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_appendix"></a>10.�Appendix</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_flume_source_catalog">10.1. Flume Source Catalog</a></span></dt><dt><span class="section"><a href="#_flume_sinks_catalog">10.2. Flume Sinks Catalog</a></span></dt><dt><span class="section"><a href="#_flume_sink_decorator_catalog">10.3. Flume Sink Decorator Catalog</a></span></dt><dt><span class="section"><a href="#_flume_environment_variables">10.4. Flume Environment Variables</a></span></dt><dt><span class="section"><a href="#_flume_site_xml_configuration_settings">10.5. flume-site.xml configuration settings</a></span></dt><dt><span class="section"><a href="#_troubleshooting">10.6. Troubleshooting</a></span></dt><dd><dl><dt><span class="section"><a href="#_what_are_the_default_ports">10.6.1. What are the default ports?</a></span></dt><dt><span class="section"><a href="#_what_versions_of_hadoop_hdfs_can_i_use_how_do_i_change_this">10.6.2. What versions of Hadoop HDFS can I use?  How do I change this?</a></span></dt><dt><span class="section"><a href="#_why_doesn_8217_t_a_flume_node_appear_on_flume_master">10.6.3. Why doesn&#8217;t a Flume node appear on Flume Master?</a></span></dt><dt><span class="section"><a href="#_why_is_the_state_of_a_flume_node_changing_rapidly">10.6.4. Why is the state of a Flume node changing rapidly?</a></span></dt><dt><span class="section"><a href="#_where_are_the_logs_for_troubleshooting_flume_itself">10.6.5. Where are the logs for troubleshooting Flume itself?</a></span></dt><dt><span class="section"><a href="#_what_can_i_do_if_i_get_node_failure_due_to_out_of_file_handles">10.6.6. What can I do if I get node failure due to out of file handles?</a></span></dt><dt><span class="section"><a href="#_failures_due_when_using_disk_failover_or_write_ahead_log">10.6.7. Failures due when using Disk Failover or Write Ahead Log</a></span></dt><dt><span class="section"><a href="#_can_i_write_data_to_amazon_s3">10.6.8. Can I write data to Amazon S3?</a></span></dt></dl></dd></dl></div><div class="section" title="10.1.�Flume Source Catalog"><div class="titlepage"><div><div><h3 class="title"><a name="_flume_source_catalog"></a>10.1.�Flume Source Catalog</h3></div></div></div><p title="Flume&#8217;s Tiered Event Sources"><b>Flume&#8217;s Tiered Event Sources.�</b>These sources and sinks are actually translated from their original
-specification into compositions of more specific lower level configurations.
-They generally have reasonable default arguments assigned by the properties
-xml file or by the Master.  The defaults can be overridden by the users.</p><div class="horizontal"><table style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">collectorSource[(<span class="emphasis"><em>port</em></span>)]</code>
-</p>
-</td><td style="" valign="top">
-<p>
-Collector source. Listens for data from
-agentSinks forwarding to port <code class="literal"><span class="emphasis"><em>port</em></span></code>.  If port is not specified, the
-node default collector TCP port, 35863.  This source registers itself
-at the Master so that its failover chains can automatically be determined.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">autoCollectorSource</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Auto collector source. Creates a logical collector
-that, when assigned to a physical node, will be included in the list of
-collectors in a failover chain. This is the collector counterpart to
-auto*Chain() sinks. See the section Automatic Failover Chains for additional
-information.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">logicalSource</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Logical Source. This source has a port assigned to it by
-the Master and listens for rpcSink formatted data.
-</p>
-</td></tr></tbody></table></div><p title="Flume&#8217;s Basic Sources"><b>Flume&#8217;s Basic Sources.�</b>These sources are untranslated and generally need all of their arguments.</p><div class="horizontal"><table style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">null</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Null source. Opens, closes, and returns null (last record) on next().
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">console</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Stdin console source.  This is for inputting events as an
-interactive user and provides features such as edit history and
-keyboard edit shortcuts.  A flume node must be started with the <code class="literal">flume
-node_nowatch</code>&#8201;&#8212;&#8201;the watchdog does not allow console input.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">stdin</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Stdin source.  This is for piping data into a flume node&#8217;s
-standard input data source.  A flume node must be started with the
-<code class="literal">flume node_nowatch</code>&#8201;&#8212;&#8201;the watchdog does not allow console input.
-WARNING: although this can be used as an interactive console, it will
-hang a flume node until a newline is entered.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">rpcSource(<span class="emphasis"><em>port</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-A remote procedure call (RPC) server that is
-configured to listen on TCP port <code class="literal"><span class="emphasis"><em>port</em></span></code>. Supports both Apache-Thrift and
-Apache-Avro RPC framework.  The type of RPC framework is specified by <code class="literal">event.rpc.type</code>
-property (THRIFT or AVRO), the default is THRIFT.  Note that same RPC framework is used for rpcSink.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">text("<span class="emphasis"><em>filename</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-One-time text file source.  One event per <code class="literal">\n</code>
-delimited line.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">tail("<span class="emphasis"><em>filename</em></span>"[, <span class="emphasis"><em>startFromEnd</em></span>=false])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Similar to Unix&#8217;s tail
-utility. One line is one event. Generates events for the entire file
-then stays open for more data, and follows filename.  (e.g. if tailing
-file "foo" and then "foo" is moved to "bar" and a new file appears
-named "foo", it will finish reading the new "bar" file and then start
-from the beginning of "foo"). If the <code class="literal">startFromEnd</code> parameter is
-false, tail will re-read from the beginning of the file.  If it is
-true, it will only start reading from the current end of file.  If the
-last line of a file does not end with a newline character (<span class="emphasis"><em>\n</em></span>), the
-<code class="literal">tail</code> source will only send an event with this last line when the
-<code class="literal">tail</code> is closed.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">multitail("<span class="emphasis"><em>filename</em></span>"[, <span class="emphasis"><em>file2</em></span> [,<span class="emphasis"><em>file3</em></span> &#8230; ] ])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Like <code class="literal">tail</code> but can
-follow multiple files concurrently.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">tailDir("<span class="emphasis"><em>dirname</em></span>"[, fileregex=".*"])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Tails all files in a
-directory <code class="literal">dirname</code> that match the specified <code class="literal">fileregex</code>.  Be careful
-and make sure because the regex argument requires java style escaping
-of <span class="emphasis"><em>\</em></span> and <span class="emphasis"><em>\"</em></span>.  For example <span class="emphasis"><em>\w+</em></span> would have to be written as
-"\\w+".  If a new file appears, it is added to the list of files to
-tail.  If pointed at a new directory, it will attempt to read all
-files that match!
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">seqfile("<span class="emphasis"><em>filename</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Read from a Hadoop sequence file formatted file,
-with <code class="literal">com.cloudera.flume.handlers.hdfs.WriteableEventKey</code> and
-<code class="literal">com.cloudera.flume.handlers.hdfs.WriteableEvent</code> values. Conveniently, this
-source can read files generated by the seqfile sink.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">syslogUdp(<span class="emphasis"><em>port</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Syslog over UDP <code class="literal"><span class="emphasis"><em>port</em></span></code>.  This is syslog compatible.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">syslogTcp(<span class="emphasis"><em>port</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Syslog over TCP <code class="literal"><span class="emphasis"><em>port</em></span></code>. This is syslog-ng compatible.
-This is a server that can listen and receive on many concurrent connections.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">syslogTcp1(<span class="emphasis"><em>port</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Syslog over TCP <code class="literal"><span class="emphasis"><em>port</em></span></code>. This is syslog-ng
-compatible.  This is only available for a single connection and then shuts
-down afterwards.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">execPeriodic("<span class="emphasis"><em>cmdline</em></span>", <span class="emphasis"><em>ms</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Execute an arbitrary program
-specified by <code class="literal"><span class="emphasis"><em>cmdline</em></span></code>.  The entire output of the execution becomes
-the body of generated messages.  <code class="literal"><span class="emphasis"><em>ms</em></span></code> specifies the number of
-milliseconds to wait before the next execution (and next
-event). Ideally the program is short lived.  This does not process
-shell pipes or redirection operations&#8201;&#8212;&#8201;for these write a script and
-use the script as the <code class="literal"><span class="emphasis"><em>cmdline</em></span></code> argument.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">execStream("<span class="emphasis"><em>cmdline</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Execute an arbitrary program specified by
-<code class="literal"><span class="emphasis"><em>cmdline</em></span></code>.  Each line outputted will become a new event.  Ideally
-the program is long lived.  This does not process shell pipes or
-redirection operations&#8201;&#8212;&#8201;for these write a script and use the script
-as the <code class="literal"><span class="emphasis"><em>cmdline</em></span></code> argument.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">exec("<span class="emphasis"><em>cmdline</em></span>"[, <span class="emphasis"><em>aggregate</em></span>=false[, <span class="emphasis"><em>restart</em></span>=false[,<span class="emphasis"><em>period</em></span>=0]]])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Execute an arbitrary program specified by <code class="literal"><span class="emphasis"><em>cmdline</em></span></code>.  If the <code class="literal"><span class="emphasis"><em>aggregate</em></span></code>
-argument is <code class="literal">true</code> entire program output is considered an event; otherwise,
-each line is considered a new event.  If the <code class="literal"><span class="emphasis"><em>restart</em></span></code> argument is <code class="literal">true</code>,
-then the program is restarted after it exits after waiting for <code class="literal"><span class="emphasis"><em>period</em></span></code>
-milliseconds. <code class="literal">execStream("foo")</code> is equivalent to <code class="literal">exec("foo", false, false,
-0)</code>. <code class="literal">execPeriodic("foo", 1000)</code> is equivalent to <code class="literal">exec("foo", true, true,
-1000)</code>
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">synth(<span class="emphasis"><em>msgCount</em></span>,<span class="emphasis"><em>msgSize</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-A source that synthetically generates
-<code class="literal"><span class="emphasis"><em>msgCount</em></span></code> random messages of size <code class="literal"><span class="emphasis"><em>msgSize</em></span></code>.  This will generate non
-printable characters.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">synthrndsize(<span class="emphasis"><em>msgCount</em></span>,<span class="emphasis"><em>minSize</em></span>,<span class="emphasis"><em>maxSize</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-A source that synthetically
-generates <code class="literal"><span class="emphasis"><em>msgCount</em></span></code> random messages of size between randomly <span class="emphasis"><em>minSize</em></span> and <span class="emphasis"><em>maxSize</em></span>.
-This will generate non printable characters.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">nonlsynth(<span class="emphasis"><em>msgCount</em></span>,<span class="emphasis"><em>msgSize</em></span>)</code>
-</p>
-</td><td style="" valign="top">
-<p>
-A source that synthetically generates
-<code class="literal"><span class="emphasis"><em>msgCount</em></span></code> random messages of size <code class="literal"><span class="emphasis"><em>msgSize</em></span></code>.  This converts all <code class="literal">'\n'</code>
-chars into <code class="literal">' '</code> chars.  This will generate non-printable characters but since
-all randomly generated <span class="emphasis"><em>\n</em></span> are converted, sources dependent on <span class="emphasis"><em>\n</em></span> as a
-record separator can get uniformly sized data.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">asciisynth(<span class="emphasis"><em>msgCount</em></span>,<span class="emphasis"><em>msgSize</em></span>)</code>
-</p>
-</td><td style="" valign="top">
-<p>
-A source that synthetically generates
-<code class="literal"><span class="emphasis"><em>msgCount</em></span></code> random messages of size <code class="literal"><span class="emphasis"><em>msgSize</em></span></code>.  This converts all <code class="literal">'\n'</code>
-chars into <code class="literal">' '</code> chars, and all non ASCII characters into printable ASCII
-characters.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">twitter("<span class="emphasis"><em>username</em></span>","<span class="emphasis"><em>pw</em></span>"[,"<span class="emphasis"><em>url</em></span>"])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-(Unsupported) A source that
-collects data from a twitter "spritzer" stream.  <code class="literal"><span class="emphasis"><em>username</em></span></code> is a twitter
-username, <code class="literal"><span class="emphasis"><em>pw</em></span></code> is the password for the user, and <code class="literal"><span class="emphasis"><em>url</em></span></code> is the url for the
-feed.  If not specified, <code class="literal">http://stream.twitter.com/1/statuses/sample.json</code> is
-used by default the <code class="literal"><span class="emphasis"><em>url</em></span></code>. See <a class="ulink" href="http://apiwiki.twitter.com/Streaming-API" target="_top">http://apiwiki.twitter.com/Streaming-API</a>-
-Documentation for more details.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">irc("<span class="emphasis"><em>server</em></span>",<span class="emphasis"><em>port</em></span>, "<span class="emphasis"><em>nick</em></span>","<span class="emphasis"><em>chan</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-(Unsupported) An IRC channel
-source.  Each line sent to the channel is a new event.  It attempts to connect
-to <code class="literal"><span class="emphasis"><em>server</em></span></code> on TCP port <code class="literal"><span class="emphasis"><em>port</em></span></code> (standard is 6667).  When it connects it
-attempts to take the nickname <code class="literal"><span class="emphasis"><em>nick</em></span></code>, and enter channel <code class="literal"><span class="emphasis"><em>chan</em></span></code> (like
-<code class="literal">#hadoop</code>   ).
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">scribe[(+<span class="emphasis"><em>port</em></span></code>)] 
-</p>
-</td><td style="" valign="top">
-<p>
-A scribe source.  This provides a network socket that
-is compatible with data generated by Facebook&#8217;s Scribe collection system.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">report[(periodMillis)]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This source polls the local physical node for its
-report every <span class="emphasis"><em>periodMillis</em></span> milliseconds and turns it into a new event.  The
-attribute names seen from the node report page are present, and the values are
-uninterpreted arrays of bytes.
-</p>
-</td></tr></tbody></table></div></div><div class="section" title="10.2.�Flume Sinks Catalog"><div class="titlepage"><div><div><h3 class="title"><a name="_flume_sinks_catalog"></a>10.2.�Flume Sinks Catalog</h3></div></div></div><div class="horizontal"><a name="id422467"></a><p class="title"><b>Table�4.�Flume&#8217;s Collector Tier Event Sinks</b></p><div class="horizontal-contents"><table summary="Flume&#8217;s Collector Tier Event Sinks" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">collectorSink("<span class="emphasis"><em>fsdir</em></span>","<span class="emphasis"><em>fsfileprefix</em></span>", <span class="emphasis"><em>rollmillis</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Collector sink.
-<code class="literal"><span class="emphasis"><em>fsdir</em></span></code> is a fs directory URI such as <code class="literal">hdfs://namenode/path</code> or <code class="literal">file:///
-path</code>.  <code class="literal"><span class="emphasis"><em>fsfileprefix</em></span></code> is a file name prefix for outputted files. Both of
-these can use escape sequences documented to bucket data as documented in the
-<span class="strong"><strong>Output Bucketing</strong></span> section. <code class="literal"><span class="emphasis"><em>rollmillis</em></span></code> is the number of milliseconds
-between when a HDFS file should be rolled (opened and closed).  The format for
-data outputted by collectors is specified by the
-<code class="literal">flume.collector.output.format</code> property.
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"><div class="horizontal"><a name="id422568"></a><p class="title"><b>Table�5.�Flume&#8217;s Agent Tier Event Sinks</b></p><div class="horizontal-contents"><table summary="Flume&#8217;s Agent Tier Event Sinks" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">agentSink[("<span class="emphasis"><em>machine</em></span>"[, <span class="emphasis"><em>port</em></span>])]</code>  
-</p>
-</td><td style="" valign="top">
-<p>
-Defaults to <code class="literal">agentE2ESink</code>
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">agentE2ESink[("<span class="emphasis"><em>machine</em></span>"[, <span class="emphasis"><em>port</em></span>])]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Agent sink with write ahead log and
-end-to-end ack.  Optional arguments specify a <code class="literal"><span class="emphasis"><em>machine</em></span></code>, and the TCP
-<code class="literal"><span class="emphasis"><em>port</em></span></code> pointing to a <code class="literal">collectorSource</code>.  If none is specified, the values
-specified by the <code class="literal">flume.collector.event.host</code> and the <code class="literal">flume.collector.port</code>
-properties will be used.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">agentDFOSink[("<span class="emphasis"><em>machine</em></span>"[, <span class="emphasis"><em>port</em></span>])]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-DiskFailover Agent sink that stores
-to local disk on detected failure. This sink periodically checks with the
-<code class="literal"><span class="emphasis"><em>machine:port</em></span></code> and resends events if becomes alive again. Optional arguments
-specify a <code class="literal"><span class="emphasis"><em>machine</em></span></code>, and the TCP <code class="literal"><span class="emphasis"><em>port</em></span></code> pointing to a <code class="literal">collectorSource</code>.
-If none is specified, the values specified by the <code class="literal">flume.collector.event.host</code>
-and the <code class="literal">flume.collector.port</code> properties will be used.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">agentBESink[("<span class="emphasis"><em>machine</em></span>"[, <span class="emphasis"><em>port</em></span>])]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-BestEffort Agent sink.  This drops
-messages on failures and continues sending.  Optional arguments specify a
-<code class="literal"><span class="emphasis"><em>collector</em></span></code>, and the TCP <code class="literal"><span class="emphasis"><em>PORT</em></span></code> pointing to a <code class="literal">collectorSource</code>.  If none
-is specified, the values specified by the <code class="literal">flume.collector.event.host</code> and the
-<code class="literal">flume.collector.port</code> properties will be used.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">agentE2EChain("<span class="emphasis"><em>m1</em></span>[:_p1_]"[, "<span class="emphasis"><em>m2</em></span>[:_p2_]"[,&#8230;]])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Agent sink
-with write-ahead log and end-to-end ack and collector failover
-chains. <code class="literal"><span class="emphasis"><em>m1:p1</em></span></code> specifies a machine and optional port of the primary
-default collector.  If all failovers are exhausted due to failures,
-and since data is already durable locally, it will back off attempts
-to send down stream.  Optional arguments specify a list of failover
-machine:port pairs in a ranked order.  If a primary collector is not
-responding, the backups are used.  The primary collectors are checked
-periodically to see if they have come back up.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">agentDFOChain("<span class="emphasis"><em>m1</em></span>[:_p1_]"[, "<span class="emphasis"><em>m2</em></span>[:_p2_]"[,&#8230;]])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-DiskFailover
-Agent sink that first attempts to fail over to other
-collectors. <code class="literal"><span class="emphasis"><em>m1:p1</em></span></code> specifies a machine and optional port of the
-primary default collector.  If all failovers are exhausted due to
-failures, it will store to local disk.  Optional arguments specify a
-list of failover machine:port pairs in a ranked order.  If a primary
-collector is not responding, the backups are used.  The primary
-collectors are checked periodically to see if they have come back up.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">agentBEChain("<span class="emphasis"><em>m1</em></span>[:_p1_]"[, "<span class="emphasis"><em>m2</em></span>[:_p2_]"[,&#8230;]])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-BestEffort
-Agent sink with collector failover chains.  <code class="literal"><span class="emphasis"><em>m1:p1</em></span></code> specifies a
-machine and optional port of the primary default collector.  If all
-failovers are exhausted due to failures, this drops messages. Optional
-arguments specify a <code class="literal"><span class="emphasis"><em>collector</em></span></code>, and the TCP <code class="literal"><span class="emphasis"><em>port</em></span></code> of the
-collector.  If none is specified, the values specified by the
-<code class="literal">flume.collector.event.host</code> and the <code class="literal">flume.collector.port</code> properties
-will be used.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">autoE2EChain</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This sink is an <code class="literal">agentE2EChain</code> that has failover nodes
-populated automatically by the master.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">autoDFOChain</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This sink is an <code class="literal">agentDFOChain</code> that has failover nodes
-populated automatically by the master.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">autoBEChain</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This sink is an <code class="literal">agentBEChain</code> that has failover nodes
-populated automatically by the master.
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"><div class="variablelist" title="Flume&#8217;s Logical Sinks"><p class="title"><b>Flume&#8217;s Logical Sinks</b></p><dl><dt><span class="term">
-<code class="literal">logicalSink("<span class="emphasis"><em>logicalnode</em></span>")</code> 
-</span></dt><dd>
-This sink creates an rpcSink that is
-assigned a host and IP based on the name of a logical node.  This information
-is maintained and automatically selected by the master.
-</dd></dl></div><div class="horizontal"><a name="id423066"></a><p class="title"><b>Table�6.�Flume&#8217;s Basic Sinks</b></p><div class="horizontal-contents"><table summary="Flume&#8217;s Basic Sinks" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">null</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Null sink. Events are dropped
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">console[("<span class="emphasis"><em>formatter</em></span>")]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Console sink.  Display events to process&#8217;s
-stdout using the optionally specified output formatter.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">text("<span class="emphasis"><em>txtfile</em></span>"[,"<span class="emphasis"><em>formatter</em></span>"])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Textfile sink.  Write to text file
-txtfile, using an optionally specified <code class="literal"><span class="emphasis"><em>formatter</em></span></code>.  If a file already
-exists, this sink will attempt to overwrite it.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">seqfile("filename")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Seqfile sink.  Write to a Hadoop sequence file
-formatted file, with <code class="literal">com.cloudera.flume.handlers.hdfs.WriteableEventKey</code> keys
-and <code class="literal">com.cloudera.flume.handlers.hdfs.WriteableEvent</code> values.  If a file
-already exists, this sink will attempt to overwrite it.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">dfs("<span class="emphasis"><em>hdfspath</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Hadoop dfs seqfile sink.  Write to a dfs path
-in Flume- specific Hadoop seqfile record format. The <code class="literal"><span class="emphasis"><em>hdfspath</em></span></code> can
-use escape sequences to bucket data as documented in the <span class="strong"><strong>Output
-Bucketing</strong></span> section.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">customdfs("<span class="emphasis"><em>hdfspath</em></span>"[, "<span class="emphasis"><em>format</em></span>"])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Hadoop dfs formatted file
-sink.  The <span class="emphasis"><em>hdfspath</em></span> string is <span class="strong"><strong>not</strong></span> escaped. The output format of
-writes to a dfs path in using specified output <span class="emphasis"><em>format</em></span>.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-+escapedCustomDfs("<span class="emphasis"><em>hdfspath</em></span>", "<span class="emphasis"><em>file</em></span>", "<span class="emphasis"><em>format</em></span>") 
-</p>
-</td><td style="" valign="top">
-<p>
-Hadoop dfs formatted
-file sink.  The <span class="emphasis"><em>hdfspath</em></span> string is escaped and events will get written to
-particular directories and filenames based on this string. The output format
-of writes to a dfs path in using specified output <span class="emphasis"><em>format</em></span>. The <code class="literal"><span class="emphasis"><em>hdfspath</em></span></code>
-can use escape sequences documented to bucket data as documented in the
-<span class="strong"><strong>Output Bucketing</strong></span> section.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">rpcSink("<span class="emphasis"><em>host</em></span>"[, <span class="emphasis"><em>port</em></span>])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-A remote procedure call (RPC) sink that is
-configured to send to machine <code class="literal"><span class="emphasis"><em>host</em></span></code> on TCP port <code class="literal"><span class="emphasis"><em>port</em></span></code>. Default port is 35861
-and can be overridden by setting the <code class="literal">flume.collector.event.port</code> property.
-Supports both Apache-Thrift and Apache-Avro RPC framework.
-The type of RPC framework is specified by <code class="literal">event.rpc.type</code> property (THRIFT or AVRO),
-the default is THRIFT.  Note that same RPC framework is used for rpcSource.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">syslogTcp("<span class="emphasis"><em>host</em></span>"[,<span class="emphasis"><em>port</em></span>])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Syslog TCP sink.  Write to host "host" on
-port "port" in syslog over TCP format (syslog-ng compatible). Default port is
-TCP 514.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">irc("<span class="emphasis"><em>host</em></span>",<span class="emphasis"><em>port</em></span>, "<span class="emphasis"><em>nick</em></span>", "<span class="emphasis"><em>chan</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-(Unsupported) An IRC channel
-sink.  Each event is sent to the channel as a line.  It attempts to connect to
-<code class="literal"><span class="emphasis"><em>server</em></span></code> on TCP port <code class="literal"><span class="emphasis"><em>port</em></span></code>.  When it connects it attempts to take the
-nickname <code class="literal"><span class="emphasis"><em>nick</em></span></code>, and enter channel <code class="literal"><span class="emphasis"><em>chan</em></span></code> (like <code class="literal">#hadoop</code>).
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"></div><div class="section" title="10.3.�Flume Sink Decorator Catalog"><div class="titlepage"><div><div><h3 class="title"><a name="_flume_sink_decorator_catalog"></a>10.3.�Flume Sink Decorator Catalog</h3></div></div></div><div class="horizontal"><a name="id423502"></a><p class="title"><b>Table�7.�Flume&#8217;s Sink Decorators</b></p><div class="horizontal-contents"><table summary="Flume&#8217;s Sink Decorators" style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">nullDeco</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This is a decorator that just passes data through to its child sink.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">writeAhead(&#8230;)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Write-ahead decorator.  Provides durability by writing
-events to disk before sending them.  This can be used as a buffering mechanism&#8201;&#8212;&#8201;receive and send are decoupled in different threads.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">ackedWriteAhead[(<span class="emphasis"><em>maxmillis</em></span>)]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Write-ahead decorator that adds
-acknowledgement tags and checksums to events.  Provides durability by writing
-events to disk before sending them.  This can be used as a buffering mechanism&#8201;&#8212;&#8201;receive and send are decoupled in different threads.  This generates and
-tracks groups of events, and also notifies other components to check for
-acknowledgements.  These checks for retries are done where there is an
-exponential backoff that can top out at <span class="emphasis"><em>maxmillis</em></span> milliseconds.  The default
-value for <span class="emphasis"><em>maxmillis</em></span> is <code class="literal">flume.agent.logdir.maxage</code> property.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">diskFailover[(<span class="emphasis"><em>maxmillis</em></span>)]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Disk failover decorator.  Events that enter
-this decorator are sent to its sub sink.  In the event of a down stream error,
-data is written to disk, and periodically these disk-buffered events are
-retried.  These checks for retries are done where there is an exponential
-backoff that can top out at <span class="emphasis"><em>maxmillis</em></span> milliseconds.  The default value for
-<span class="emphasis"><em>maxmillis</em></span> is <code class="literal">flume.agent.logdir.maxage</code> property.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">ackInjector</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This decorator injects an extra ack group start message on
-open, tags appended events with an ack tag, and injects an extra ack group end
-message.  These tags contain a checksum, for all the bodies of the events that
-pass through the <code class="literal">ackInjector</code>.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">ackChecker</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This decorator tracks ack group start, end, and checksum
-values inserted by <code class="literal">ackInjector</code>.  If a group has arrived and its checksum is
-correct, it sends notifications to other components.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">lazyOpen</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This decorator tracks open/closed state of the sub sink but does
-not actually open the sink until an append is called.  Thus if a this
-decorator is opened and closed without any appends, the sub sink is never
-opened.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">insistentOpen[(<span class="emphasis"><em>max</em></span>[<span class="emphasis"><em>init</em></span>[,<span class="emphasis"><em>cumulativeMax</em></span>]],)]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-An insistent open
-attempts to open its subsink multiple times until it succeeds with the
-specified backoff properties.  This is useful for starting a network client up
-when the network server may not yet be up.  When an attempt to open the
-subsink fails, this exponentially backs off and then retries the open.  <span class="emphasis"><em>max</em></span>
-is the max number of millis per backoff (default is Integer.MAX_VALUE).
-<span class="emphasis"><em>init</em></span> is the initial number of millis to back off on the first encountered
-failure (default 1000). <span class="emphasis"><em>cumulativeMax</em></span> is the maximum backoff allowed from a
-single failure before an exception is forwarded (default is
-Integer.MAX_VALUE).  Note that this synchronously blocks the open call until
-the open succeeds or fails after <span class="emphasis"><em>cumulativeMax</em></span> millis.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">stubbornAppend</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-A stubborn append normally passes through append
-operations to its subsink.  It catches the first exception that a subsink&#8217;s
-append method triggers, and then closes, opens, and reappends to the subsink.
-If this second attempt fails, it throws an exception. This is useful in
-conjunction with network sinks where connections can be broken.  The open/
-close retry attempt is often sufficient to re-establish the connection.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">value("<span class="emphasis"><em>attr</em></span>","<span class="emphasis"><em>value</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The value decorator adds a new metadata
-attribute <span class="emphasis"><em>attr</em></span> with the value <span class="emphasis"><em>value</em></span>.  Agents can mark their data with
-specific tags for later demultiplexing.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">mask("<span class="emphasis"><em>attr1</em></span>"[,"<span class="emphasis"><em>attr2</em></span>", &#8230;])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The mask decorator outputs inputted
-events that are modified so that all metadata <span class="strong"><strong>except</strong></span> the attributes
-specified pass through.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">select("<span class="emphasis"><em>attr1</em></span>"[,"<span class="emphasis"><em>attr2</em></span>", &#8230;])</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The select decorator outputs inputted
-events that are modified so that <span class="strong"><strong>only</strong></span> the metadata attributes specified pass
-through.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">format("<span class="emphasis"><em>pattern</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The format decorator outputs inputted events that are
-modified so that their bodies are replaced with an escaped version of the
-<code class="literal"><span class="emphasis"><em>pattern</em></span></code> argument.  Since checksums rely on body data, this should only be
-used on unreliable flows or reporting flows. Inappropriate use may result in
-message loss.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">regex("<span class="emphasis"><em>regex</em></span>",idx,"<span class="emphasis"><em>attr</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The regex decorator applies the regular
-expression <span class="emphasis"><em>regex</em></span>, extracts the <span class="emphasis"><em>idx</em></span> th capture group, and writes this value
-to the <span class="emphasis"><em>attr</em></span> attribute.  The regex must use java-style escaping.  Thus a
-regexs that want to use the <code class="literal">\d</code> macro need to be specified as "\\d".
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">split("<span class="emphasis"><em>regex</em></span>",idx,"<span class="emphasis"><em>attr</em></span>")</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The split decorator uses the
-regular expression <span class="emphasis"><em>regex</em></span> to split the body into tokens (not
-including the splitter value). The <span class="emphasis"><em>idx</em></span> is then written as the value
-to the <span class="emphasis"><em>attr</em></span> attribute.  The regex must use java-style escaping.
-Thus, a regex that wants to use the <code class="literal">\d</code> macro must be specified as
-"\\d".
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">batch(<span class="emphasis"><em>n</em></span>,<span class="emphasis"><em>maxlatency</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-buffers <span class="emphasis"><em>n</em></span> events and then sends one
-aggregate event.  If <span class="emphasis"><em>maxlatency</em></span> millis have passed, all current
-buffered events are sent out as an aggregate event.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">unbatch</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Unbatch takes an aggregate event generated by batch, splits it,
-and then forwards its original events.  If an event is not an aggregate it is
-just forwarded.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">gzip</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-gzips a serialized event.  This is useful when used in conjunction
-with aggregate events.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">gunzip</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-gunzip&#8217;s a gzip&#8217;ed event.  If the event is not a gzip event, it is
-just forwarded.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">intervalSampler(<span class="emphasis"><em>n</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Interval sampler. Every <code class="literal"><span class="emphasis"><em>n</em></span></code> th event gets
-forwarded.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">probSampler(<span class="emphasis"><em>p</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Probability sampler. Every event has a probability <span class="emphasis"><em>p</em></span>
-(where 0.0 &#8804; <span class="emphasis"><em>p</em></span> &#8804; 1.0) chance of being forwarded.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">reservoirSampler(<span class="emphasis"><em>k</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Reservoir sampler.  When flushed, at most <span class="emphasis"><em>k</em></span>
-events are forwarded.  If more than <span class="emphasis"><em>k</em></span> elements have entered this decorator,
-exactly <span class="emphasis"><em>k</em></span> events are forwarded.  All events that pass through have the same
-probability of being selected.  NOTE: This will reorder the events being sent.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">delay(<span class="emphasis"><em>ms</em></span>)</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-adds a <span class="emphasis"><em>ms</em></span> millisecond delay before forwarding events down
-the pipeline.  This blocks and prevents other events from entering the
-pipeline.  This is useful for workload simulation in conjunction with
-<code class="literal">asciisynth</code> sources.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">choke[(<span class="emphasis"><em>choke-id</em></span>)]</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-Limits the transfer rate of data going into the sink.
-The <code class="literal">choke-id</code> should have been registered on the physical node where this decorator is being created using the <code class="literal">setChokeLimit</code> command.
-Refer to <span class="strong"><strong>Limiting Data Transfer Rate between Source-Sink pairs</strong></span> section for more details.
-</p>
-</td></tr></tbody></table></div></div><br class="horizontal-break"></div><div class="section" title="10.4.�Flume Environment Variables"><div class="titlepage"><div><div><h3 class="title"><a name="_flume_environment_variables"></a>10.4.�Flume Environment Variables</h3></div></div></div><p>This section describes several environment variables that affect how
-Flume operates.  The <code class="literal">flume</code> script in <code class="literal">./bin</code> uses these system
-environment variables.  Many of these variables are set by the
-<code class="literal">flume-daemon.sh</code> script used when flume is run as a daemon.</p><div class="horizontal"><table style="border: none;"><colgroup><col><col></colgroup><tbody valign="top"><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_PID_DIR</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The directory where a flume node or flume
-master will drop pid files corresponding to the daemon process.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_CLASSPATH</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The custom java classpath environment variable
-additions you want flume to run with.  These values are prepended to
-the normal Flume generated CLASSPATH.  WARNING: The <code class="literal">flume</code> script
-overwrites the standard CLASSPATH when it is executed.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_LOG_DIR</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The directory where debugging logs
-generated by the flume node or flume master are written.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_LOGFILE</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-This sets the suffix for the logfiles generated by
-flume node or flume master.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_ROOT_LOGGER</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The log4j logging setting for the executing
-command.  By default it is "INFO,console".
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">ZOOKEEPER_ROOT_LOGGER</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The log4j logging setting for a master&#8217;s
-embedded zookeeper server&#8217;s logs.  By default it is "ERROR,console".
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">WATCHDOOG_ROOT_LOGGER</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The log4j logging setting for the logging
-the watchdog that wraps flume nodes and flume masters generates.  By
-default it is "INFO,console".
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_CONF_DIR</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The directory where the <code class="literal">flume-site.xml</code> and
-<code class="literal">flume-conf.xml</code> files flume node and flume master will use reside.
-This defaults to <code class="literal">./conf</code> if a <code class="literal">./conf/flume-conf.xml</code> file is found,
-or to <code class="literal">/etc/flume/conf/</code> if it <code class="literal">flume-conf.xml</code> is found there.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">HADOOP_HOME</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-The directory where Hadoop jars are expected to be
-found.  If not specified it will use jars found in <code class="literal">/usr/lib/hadoop</code>
-or <code class="literal">./lib/</code>.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_DEVMODE</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-If this value is set to "true" the <code class="literal">./libbuild</code>
-jars which include ant jars required to compile JSP servlets will be
-included in the CLASSPATH.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_VERBOSE</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-If it this is toggled, the flume script will print
-out the command line being executed.
-</p>
-</td></tr><tr><td style="" valign="top">
-<p>
-<code class="literal">FLUME_VERBOSE_JAVA</code> 
-</p>
-</td><td style="" valign="top">
-<p>
-If this is toggled along with FLUME_VERBOSE,
-the "-verbose" flag will be passed to the JVM running flume.
-</p>
-</td></tr></tbody></table></div></div><div class="section" title="10.5.�flume-site.xml configuration settings"><div class="titlepage"><div><div><h3 class="title"><a name="_flume_site_xml_configuration_settings"></a>10.5.�flume-site.xml configuration settings</h3></div></div></div></div><div class="section" title="10.6.�Troubleshooting"><div class="titlepage"><div><div><h3 class="title"><a name="_troubleshooting"></a>10.6.�Troubleshooting</h3></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_what_are_the_default_ports">10.6.1. What are the default ports?</a></span></dt><dt><span class="section"><a href="#_what_versions_of_hadoop_hdfs_can_i_use_how_do_i_change_this">10.6.2. What versions of Hadoop HDFS can I use?  How do I change this?</a></span></dt><dt><span class="section"><a href="#_why_doesn_8217_t_a_flume_node_appear_on_flume_master">10.6.3. Why doesn&#8217;t a Flume node appear on Flume Master?</a></span></dt><dt><span class="section"><a href="#_why_is_the_state_of_a_flume_node_changing_rapidly">10.6.4. Why is the state of a Flume node changing rapidly?</a></span></dt><dt><span class="section"><a href="#_where_are_the_logs_for_troubleshooting_flume_itself">10.6.5. Where are the logs for troubleshooting Flume itself?</a></span></dt><dt><span class="section"><a href="#_what_can_i_do_if_i_get_node_failure_due_to_out_of_file_handles">10.6.6. What can I do if I get node failure due to out of file handles?</a></span></dt><dt><span class="section"><a href="#_failures_due_when_using_disk_failover_or_write_ahead_log">10.6.7. Failures due when using Disk Failover or Write Ahead Log</a></span></dt><dt><span class="section"><a href="#_can_i_write_data_to_amazon_s3">10.6.8. Can I write data to Amazon S3?</a></span></dt></dl></div><div class="section" title="10.6.1.�What are the default ports?"><div class="titlepage"><div><div><h4 class="title"><a name="_what_are_the_default_ports"></a>10.6.1.�What are the default ports?</h4></div></div></div><p>TCP ports are used in all situations.</p><div class="informaltable"><table style="border-collapse: collapse;border-top: 0.5pt solid ; border-bottom: 0.5pt solid ; border-left: 0.5pt solid ; border-right: 0.5pt solid ; " width="40%"><colgroup><col><col><col></colgroup><tbody><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>node collector port</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.collector.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>35853+</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>node status web server</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.node.http.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>35862+</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>master status web server</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.http.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>35871</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>master heartbeat port</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.heartbeat.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>35872</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>master admin/shell port</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.admin.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>35873</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>master gossip port</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.gossip.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>35890</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>master &#8594; zk port</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.zk.client.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>3181</p></td></tr><tr><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p>zk &#8594; zk quorum port</p></td><td style="border-right: 0.5pt solid ; border-bottom: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.zk.server.quorum.port</code></p></td><td style="border-bottom: 0.5pt solid ; " align="left" valign="top"><p>3182</p></td></tr><tr><td style="border-right: 0.5pt solid ; " align="left" valign="top"><p>zk &#8594; zk election port</p></td><td style="border-right: 0.5pt solid ; " align="left" valign="top"><p><code class="literal">flume.master.zk.server.election.port</code></p></td><td style="" align="left" valign="top"><p>3183</p></td></tr></tbody></table></div></div><div class="section" title="10.6.2.�What versions of Hadoop HDFS can I use? How do I change this?"><div class="titlepage"><div><div><h4 class="title"><a name="_what_versions_of_hadoop_hdfs_can_i_use_how_do_i_change_this"></a>10.6.2.�What versions of Hadoop HDFS can I use?  How do I change this?</h4></div></div></div><p>Currently, there are constraints writing to HDFS.  A Flume node
-can only write to one version of Hadoop.  Although Hadoop&#8217;s HDFS API has been
-fairly stable, HDFS clients are only guaranteed to be wire compatible with the
-same major version of HDFS.  Cloudera&#8217;s testing used Hadoop HDFS 0.20.x and HDFS
-0.18.x.  They are API compatible so all that is necessary to switch versions
-is to swap out the Hadoop jar and restart the node that will write to the
-other Hadoop version.</p><p>You still need to configure this instance of Hadoop so that it talks to the
-correct HDFS nameNode.  You configure the Hadoop client settings (such as
-pointers to the name node) the same way as with Hadoop dataNodes or worker
-nodes&#8201;&#8212;&#8201;modify and use <code class="literal">conf/core-site.xml</code>.</p><p>By default, Flume checks for a Hadoop jar in <code class="literal">/usr/lib/hadoop</code>.  If it is not
-present, it defaults to a jar found in its own lib directory, <code class="literal">/usr/lib/flume/
-lib</code>.</p></div><div class="section" title="10.6.3.�Why doesn&#8217;t a Flume node appear on Flume Master?"><div class="titlepage"><div><div><h4 class="title"><a name="_why_doesn_8217_t_a_flume_node_appear_on_flume_master"></a>10.6.3.�Why doesn&#8217;t a Flume node appear on Flume Master?</h4></div></div></div><p>If a node does not show up on the Master, you should first check to see if the
-node is running.</p><pre class="screen"># jps | grep FlumeNode</pre><p>You can also check logs found in <code class="literal">/usr/lib/flume/logs/flume-flume-node*.log</code>.</p><p>Common errors are error messages or warnings about being unable to contact
-the masters.  This could be due to host misconfiguration, port
-misconfiguration (35872 is the default heartbeat port), or fire walling
-problems.</p><p>Another possible error is to have a permissions problems with the local
-machine&#8217;s write-ahead log directory.  On an out-of-the-box setup, this is in
-the <code class="literal">/tmp/flume/agent</code> directory).  If a Flume Node is ever run as a user
-other than <code class="literal">flume</code>, (especially if it was run as <code class="literal">root</code>), the directory needs
-to be either deleted or its contents must have its permissions modified to
-allow Flume to use it.</p></div><div class="section" title="10.6.4.�Why is the state of a Flume node changing rapidly?"><div class="titlepage"><div><div><h4 class="title"><a name="_why_is_the_state_of_a_flume_node_changing_rapidly"></a>10.6.4.�Why is the state of a Flume node changing rapidly?</h4></div></div></div><p>Nodes by default start and choose their <code class="literal">hostname</code> as their physical node
-name.  Physical nodes names are supposed to be unique.  Unexpected results may
-occur if multiple physical nodes are assigned the same name.</p></div><div class="section" title="10.6.5.�Where are the logs for troubleshooting Flume itself?"><div class="titlepage"><div><div><h4 class="title"><a name="_where_are_the_logs_for_troubleshooting_flume_itself"></a>10.6.5.�Where are the logs for troubleshooting Flume itself?</h4></div></div></div><p>On Ubuntu-based installations, logs are written to <code class="literal">/usr/lib/logs/</code>.</p><div class="variablelist"><dl><dt><span class="term">
-Master logs   
-</span></dt><dd>
-<code class="literal">/usr/lib/logs/flume-flume-master-<span class="emphasis"><em>host</em></span>.log</code>
-</dd><dt><span class="term">
-Master stdout 
-</span></dt><dd>
-<code class="literal">/usr/lib/logs/flume-flume-master-<span class="emphasis"><em>host</em></span>.out.*</code>
-</dd><dt><span class="term">
-Node logs 
-</span></dt><dd>
-<code class="literal">/usr/lib/ logs/flume-flume-node-<span class="emphasis"><em>host</em></span>.log</code>
-</dd><dt><span class="term">
-Node stdout 
-</span></dt><dd>
-<code class="literal">/usr/lib/logs/flume-flume- node-<span class="emphasis"><em>host</em></span>.out.*</code>
-</dd></dl></div></div><div class="section" title="10.6.6.�What can I do if I get node failure due to out of file handles?"><div class="titlepage"><div><div><h4 class="title"><a name="_what_can_i_do_if_i_get_node_failure_due_to_out_of_file_handles"></a>10.6.6.�What can I do if I get node failure due to out of file handles?</h4></div></div></div><p>There are two limits in Linux&#8201;&#8212;&#8201;the max number of allowable open files
-(328838), and max number of allowable open files for a user (default 1024).
-Sockets are file handles so this limits the number of open TCP connections
-available.</p><p>On Ubuntu, need to add a line to <code class="literal">/etc/security/limits.conf</code></p><pre class="screen">&lt;user&gt; hard nofile 10000</pre><p>The user should also add the following line to a <code class="literal">~/.bash_profile</code> to raise
-the limit to the hard limit.</p><pre class="screen">ulimit -n 10000</pre></div><div class="section" title="10.6.7.�Failures due when using Disk Failover or Write Ahead Log"><div class="titlepage"><div><div><h4 class="title"><a name="_failures_due_when_using_disk_failover_or_write_ahead_log"></a>10.6.7.�Failures due when using Disk Failover or Write Ahead Log</h4></div></div></div><p>Flume currently relies on the file system for logging mechanisms. You must
-make sure that the user running Flume has permissions to write to the
-specified logging directory.</p><p>Currently the default is to write to /tmp/flume.  In a production system you
-should write to a directory that does not automatically get deleted on reboot.</p></div><div class="section" title="10.6.8.�Can I write data to Amazon S3?"><div class="titlepage"><div><div><h4 class="title"><a name="_can_i_write_data_to_amazon_s3"></a>10.6.8.�Can I write data to Amazon S3?</h4></div></div></div><p>Yes.  In the collector sink or dfs sinks you can use the <code class="literal">s3n://</code> or <code class="literal">s3://</code>
-prefixes to write to S3 buckets.</p><p>First you must add some jars to your CLASSPATH in order to support s3 writing.
-Here is a set that Cloudera has tested (other versions will likely work as well):</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-commons-codec-1.3.jar
-</li><li class="listitem">
-commons-httpclient-3.0.1.jar
-</li><li class="listitem">
-jets3t-0.6.1.jar
-</li></ul></div><p><code class="literal">s3n</code> uses the native s3 file system and has some limitations on individual
-file sizes.  Files written with this method are compatible with other programs
-like <code class="literal">s3cmd</code>.</p><p><code class="literal">s3</code> writes using an overlay file system must go through Hadoop for file
-system interaction.</p></div></div></div><div class="glossary" title="Glossary"><div class="titlepage"><div><div><h2 class="title"><a name="_glossary"></a>Glossary</h2></div></div></div><div class="variablelist"><dl><dt><span class="term">
-Agent
-</span></dt><dd>
-A Flume node located at the start of a flow that captures data from
-external sources, ready for feeding downstream.
-</dd><dt><span class="term">
-Collector
-</span></dt><dd>
-A Flume node located at the end of a flow that delivers data to its eventual destination.
-</dd><dt><span class="term">
-Flow
-</span></dt><dd>
-A set of nodes wired together in sequence which together process data
-from a single source into its eventual destination.
-</dd><dt><span class="term">
-Master
-</span></dt><dd>
-A service that controls the configuration of all nodes, and to
-which all nodes report.
-</dd><dt><span class="term">
-Sink
-</span></dt><dd>
-The place where a node sends its data after all processing is done.
-</dd><dt><span class="term">
-Source
-</span></dt><dd>
-The place where a node gets its data stream.
-</dd></dl></div></div><div class="section" title="11.�Versions"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a name="_versions"></a>11.�Versions</h2></div></div></div><div class="toc"><dl><dt><span class="section"><a href="#_history">11.1. history</a></span></dt></dl></div><div class="section" title="11.1.�history"><div class="titlepage"><div><div><h3 class="title"><a name="_history"></a>11.1.�history</h3></div></div></div><div class="variablelist"><dl><dt><span class="term">
-v0.9.2 11/15/10
-</span></dt><dd>
-Support for all hadoop supported compression codecs.
-Avro RPC support.  Flexible throughput thottling.  Useful error
-messsages in shell. Upgrades to Thrift RPC support.  Performance and
-robustness improvements in tail and exec.
-</dd><dt><span class="term">
-v0.9.1u1 10/12/10
-</span></dt><dd>
-Added write support for Kerberized HDFS. Flume
-cookbook.
-</dd><dt><span class="term">
-v0.9.1 8/9/10
-</span></dt><dd>
-Improved error messages and visibility of property
-configuration values.  First external contributions.  Fixed
-reconfiguration hangs.  Improved implementing plugins documentation.
-Updated scribe and syslog support.  Compression on output files.
-</dd><dt><span class="term">
-v0.9 6/29/10 
-</span></dt><dd>
-metrics and reporting framework, logical nodes+logical
-names abstraction, wal/dfo isolation by flow, transformation-based
-high level sinks.  Open source and initial push to github.
-</dd><dt><span class="term">
-v0.3 3/31/10 
-</span></dt><dd>
-ZK based master/multi-master, automatic failovers for
-data and control planes.  flume shell. deb/rpm packaging.
-</dd><dt><span class="term">
-v0.2 1/21/09 
-</span></dt><dd>
-Different reliability modes: WAL 2.0, DFO, Best
-effort. Output file escaping/bucketing.  Proliferation of many sink
-and decorators.
-</dd><dt><span class="term">
-v0.1 11/23/09 
-</span></dt><dd>
-First installation deployment, users tests.
-</dd><dt><span class="term">
-v0.0 9/21/09 
-</span></dt><dd>
-First cut with current architecture (centralized
-master, configuration language, web interface.)  First version of WAL.
-Simple visualizations, samplers, thrift based rpc.
-</dd><dt><span class="term">
-pre history 7/21/09 
-</span></dt><dd><p class="simpara">
-First commit.  design, experimental
-implementations.  Initial implementation had individual agent and
-collector programs, watchdog
-</p><pre class="literallayout">     ______
-    / ___//_  ______  ____
-   / /_/ / / / /    \/ __/
-  / __/ / /_/ / / / / __/
- / / /_/\____/_/_/_/\__/
-/_/ Distributed Log Collection.</pre></dd></dl></div></div></div></div><script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script><script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script><div class="footer-text"><span align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></span><br>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div></body></html>
diff --git a/docs/UserGuide/index.txt b/docs/UserGuide/index.txt
deleted file mode 100644
index 04ff317..0000000
--- a/docs/UserGuide/index.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-
-////
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-////
-
-Flume Documentation (v{revnumber})
-==================================
-
-The following pages are the documentation for Flume v{revnumber}:
-
-- link:UserGuide/index.html[Flume User Guide]
-- link:DeveloperGuide/index.html[Flume Developer Guide]
-- link:Cookbook/index.html[Flume Cookbook]
-
diff --git a/docs/UserGuide/master-zk-internal.png b/docs/UserGuide/master-zk-internal.png
deleted file mode 100644
index b95482574b7b2691a27dbe310f86b397563dd036..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/master-zk-standalone.png b/docs/UserGuide/master-zk-standalone.png
deleted file mode 100644
index 2c4641c9f5cf01770e231c97dee4efd6be98b12a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/multi-flow.png b/docs/UserGuide/multi-flow.png
deleted file mode 100644
index adc99dd11c246fefa387f0a1d97cd94b0f0f4322..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/multiCollector.png b/docs/UserGuide/multiCollector.png
deleted file mode 100644
index 950795546412bb8cc0274eedfbf61339924a153a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/single-flow.png b/docs/UserGuide/single-flow.png
deleted file mode 100644
index 0f624630a7d98cfd2a6c9dd267f9c3caf4904523..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/singleCollector.png b/docs/UserGuide/singleCollector.png
deleted file mode 100644
index d8d1ea5f025f1d2734edc5f8c404046f0d70c459..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/sinkStatesAppending.png b/docs/UserGuide/sinkStatesAppending.png
deleted file mode 100644
index 09b31741269f08174f774058dbbd8e522774840e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/sinkStatesBuffered.png b/docs/UserGuide/sinkStatesBuffered.png
deleted file mode 100644
index fde50fd831f86f4d383185b89fb0cc16baa07a4d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/sinkStatesOpening.png b/docs/UserGuide/sinkStatesOpening.png
deleted file mode 100644
index 1305c79a17128f31a0bae74b4c022a6b1400eaa0..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/sinkStatesSimple.png b/docs/UserGuide/sinkStatesSimple.png
deleted file mode 100644
index cf55923a631a332fd2c976d26ef08750902401c3..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/sourceStatesBuffered.png b/docs/UserGuide/sourceStatesBuffered.png
deleted file mode 100644
index 9ba7295c8d1d6d3ed61ac4847d97a0555ad70ae2..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/sourceStatesSimple.png b/docs/UserGuide/sourceStatesSimple.png
deleted file mode 100644
index 5642ace13460f9778d77b1a0539f1781996ba8e0..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/tiers-console.png b/docs/UserGuide/tiers-console.png
deleted file mode 100644
index f27302dff175b1eaecdffccea42e7f65eafbe7da..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/UserGuide/tiers-hdfs.png b/docs/UserGuide/tiers-hdfs.png
deleted file mode 100644
index 1d7fa32cd3f2a08f375dee047f25e4fadccb2eda..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/breadcrumbs.xsl b/docs/breadcrumbs.xsl
deleted file mode 100644
index 98d70ba..0000000
--- a/docs/breadcrumbs.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<xsl:template name="breadcrumbs">
-  <xsl:param name="this.node" select="."/>
-  <div class="breadcrumbs">
-    <xsl:for-each select="$this.node/ancestor::*">
-      <span class="breadcrumb-link">
-        <a>
-          <xsl:attribute name="href">
-            <xsl:call-template name="href.target">
-              <xsl:with-param name="object" select="."/>
-              <xsl:with-param name="context" select="$this.node"/>
-            </xsl:call-template>
-          </xsl:attribute>
-          <xsl:apply-templates select="." mode="title.markup"/>
-        </a>
-      </span>
-      <xsl:text> &gt; </xsl:text>
-    </xsl:for-each>
-    <!-- And display the current node, but not as a link -->
-    <span class="breadcrumb-node">
-      <xsl:apply-templates select="$this.node" mode="title.markup"/>
-    </span>
-  </div>
-</xsl:template>
-
-</xsl:stylesheet>
diff --git a/docs/docbook.css b/docs/docbook.css
deleted file mode 100644
index a622c9f..0000000
--- a/docs/docbook.css
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- *
- *
- * This file was provided to this project under these terms:
- *
- * Copyright (c) 2001, 2003 The FreeBSD Documentation Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD: doc/share/misc/docbook.css,v 1.14 2008/11/21 07:28:34 keramida Exp $
- */
-
-BODY ADDRESS {
-	line-height: 1.3;
-	margin: .6em 0;
-}
-
-BODY BLOCKQUOTE {
-	margin-top: .75em;
-	line-height: 1.5;
-	margin-bottom: .75em;
-}
-
-HTML BODY {
-	margin: 1em 4% 1em 4%;
-	line-height: 1.2;
-	font-family: verdana;
-}
-
-a {
-    text-decoration: none;
-    color: #1c84bf;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-a:visited {
-    color: #597e93;
-}
-
-div.titlepage .title {
-	font-size: 26px;
-}
-
-.LEGALNOTICE {
-	font-size: small;
-	font-variant: small-caps;
-}
-
-BODY DIV {
-	margin: 0;
-}
-
-DL {
-	margin: .8em 0;
-	line-height: 1.2;
-}
-
-BODY FORM {
-	margin: .6em 0;
-}
-
-H1, H2, H3, H4, H5, H6,
-DIV.EXAMPLE P B,
-.QUESTION,
-DIV.TABLE P B,
-DIV.PROCEDURE P B {
-	color: #148BCF;
-}
-
-BODY H1, BODY H2, BODY H3, BODY H4, BODY H5, BODY H6 {
-	line-height: 1.3;
-	margin-left: 0;
-}
-
-BODY H1, H2, H3, H4, H5, H6 {
-	margin: 6px 0 0 0;
-}
-
-BODY HR {
-	border-width: 0 0 1px 0;
-	border-style: solid;
-	border-color: #0068BB;
-}
-
-BODY IMG.NAVHEADER {
-	margin: 0 0 0 -4%;
-}
-
-OL {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-BODY PRE {
-	margin: .75em 0;
-	line-height: 1.0;
-	font-family: monospace;
-}
-
-BODY TD, BODY TH {
-	line-height: 1.2;
-}
-
-UL, BODY DIR, BODY MENU {
-	margin: 0 0 0 5%;
-	line-height: 1.2;
-}
-
-HTML {
-	margin: 0; 
-	padding: 0;
-}
-
-BODY P B.APPLICATION {
-	color: #000000;
-}
-
-.navfooter td, .literal, .FILENAME {
-	color: #007a00;
-	font-family: monospace;
-}
-
-
-.GUIMENU, .GUIMENUITEM, .GUISUBMENU,
-.GUILABEL, .INTERFACE,
-.SHORTCUT, .SHORTCUT .KEYCAP {
-	font-weight: bold;
-}
-
-.GUIBUTTON {
-	background-color: #CFCFCF;
-	padding: 2px;
-}
-
-.ACCEL {
-	background-color: #F0F0F0;
-	text-decoration: underline;
-} 
-
-.PROGRAMLISTING {
-	padding: 1ex;
-	background-color: #eee;
-	border: 1px solid #ccc;
-}
-
-@media screen {  /* hide from IE3 */:/
-	a[href]:hover { background: #ffa }
-}
-
-.NOTE, .SCREEN {
-	color: #222;
-	background: #fff;
-	border: 1px solid #ccc;
-	padding: 0.4em 0.4em;
-}
-
-.SCREEN {
-        background: #eee;
-}
-
-.TIP {
-	color: #004F00;
-	background: #fff;
-	border: 1px solid green;
-	padding: 0.4em 0.4em;
-}
-
-table.CDH {
-	padding:0px 0px 8px
-}
-
-table.CDH th {
-	background: #EDF2F9;
-	border-right: 0.5pt solid #adb6cd;
-	border-bottom: 0.5pt solid #adb6cd;
-        padding:2px 8px 1px;
-}
-
-table.CDH td {
-	font-size: 12px;
-}
-
-div.breadcrumbs {
-	border-top: 2px solid #0068BB;
-        background: #EDF2F9;
-	font-style: bold;
-	font-size: small;
-	padding: 0.6em 0.6em;
-}
-
-span.breadcrumb-node {
-	font-style: italic;
-}
-
-/* Match the the breadcrumbs at the top */
-div.footer-text {
-	border-bottom: 2px solid #0068BB;
-        background: #EDF2F9;
-        font-size: small;
-        padding: 0.2em 0.2em;
-	text-align: center;
-}
-
-.IMPORTANT, .caution {
-	font-style:italic;
-	border: 1px solid #a00;
-	border-left: 12px solid #c00;
-	padding: 0.1em 1em;
-}
-
-.WARNING {
-	color: #9F1313;
-	background: #fff;
-	border: 1px solid #e59595;
-	padding: 0.4em 0.4em;
-	width: 85%;
-}
-
-.EXAMPLE {
-	width: 100%;
-}
-
-.tip, .note, .caution {
-	margin:4px 0 4px;
-}
-
-img {
-	border: 0;
-}
diff --git a/docs/footer.xsl b/docs/footer.xsl
deleted file mode 100644
index 98582c0..0000000
--- a/docs/footer.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:template name="user.footer.content">
-
-  <script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script>
-  <script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script>
-
-  <div class="footer-text">
-  <span align="center"><a href="index.html"><img src="images/home.png"
-      alt="Documentation Home" /></a></span>
-  <br/>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/header.xsl b/docs/header.xsl
deleted file mode 100644
index 7c74909..0000000
--- a/docs/header.xsl
+++ /dev/null
@@ -1,41 +0,0 @@
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                xmlns:exsl="http://exslt.org/common"
-                version="1.0"
-                exclude-result-prefixes="exsl">
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-<xsl:import href="breadcrumbs.xsl"/>
-
-<xsl:template name="user.head.content">
-</xsl:template>
-
-<xsl:template name="user.header.content">
-
-    <div style="clear:both; margin-bottom: 4px" />
-    <div align="center">
-      <a href="index.html"><img src="images/home.png"
-          alt="Documentation Home" /></a>
-    </div>
-    <span class="breadcrumbs">
-    <xsl:call-template name="breadcrumbs"/>
-    </span>
-
-</xsl:template>
-</xsl:stylesheet>
diff --git a/docs/html.xsl b/docs/html.xsl
deleted file mode 100644
index 2617dd4..0000000
--- a/docs/html.xsl
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version='1.0'?>
-
-
-<!--
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- $FreeBSD: doc/share/xsl/freebsd-html.xsl,v 1.1 2003/01/03 05:06:14 trhodes Exp $ -->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                version='1.0'
-                xmlns="http://www.w3.org/TR/xhtml1/transitional"
-                exclude-result-prefixes="#default">
-
-  <xsl:import href="header.xsl"/>
-  <xsl:import href="footer.xsl"/>
-
-  <xsl:template name="body.attributes" />
-
-  <xsl:param name="html.stylesheet" select="'docbook.css'"/>
-  <xsl:param name="use.id.as.filename" select="1"/>
-  <xsl:param name="generate.legalnotice.link" select="'1'"/>
-  <xsl:param name="generate.section.toc.level" select="100"></xsl:param>
-
-  <xsl:param name="toc.section.depth" select="10"/>
-  <xsl:param name="section.autolabel" select="1"/>
-  <xsl:param name="section.label.includes.component.label" select="1"/>
-  <xsl:param name="chunk.section.depth" select="100"></xsl:param>
-  <xsl:param name="chunk.first.sections" select="1"></xsl:param>
-  <xsl:param name="navig.showtitles" select="1"></xsl:param>
-
-  <xsl:param name="admon.graphics" select="1"></xsl:param>
-  <xsl:param name="admon.graphics.extension">.png</xsl:param>
-  <xsl:param name="admon.graphics.path">images/</xsl:param>
-
-  <xsl:param name="navig.graphics" select="1"></xsl:param>
-  <xsl:param name="navig.graphics.extension">.png</xsl:param>
-  <xsl:param name="navig.graphics.path">images/</xsl:param>
-
-  <xsl:param name="header.rule" select="0"></xsl:param>
-  <xsl:param name="footer.rule" select="0"></xsl:param>
-  <xsl:param name="suppress.header.navigation" select="1"></xsl:param>
-
-  <xsl:param name="generate.index" select="1"></xsl:param>
-
-  <xsl:param name="spacing.paras" select="0"></xsl:param>
-  <xsl:param name="html.cleanup" select="1"></xsl:param>
-
-  <xsl:param name="table.borders.with.css" select="1"></xsl:param>
-  <!-- xsl:param name="id.warnings" select="1"></xsl:param -->
-
-  <xsl:param name="generate.toc">
-appendix  toc,title
-article   toc,title,figure,equation
-book      toc,title,figure,example,equation
-chapter   toc,title
-part      toc,title
-preface   toc,title
-qandadiv  toc
-qandaset  toc
-reference toc,title
-sect1     toc
-sect2     toc
-sect3     toc
-sect4     toc
-sect5     toc
-section   toc
-set       toc,title
-   </xsl:param>
-
-</xsl:stylesheet>
diff --git a/docs/images/README b/docs/images/README
deleted file mode 100644
index f12b2a7..0000000
--- a/docs/images/README
+++ /dev/null
@@ -1,5 +0,0 @@
-Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook
-icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency
-from the Jimmac icons to get round MS IE and FOP PNG incompatibilies.
-
-Stuart Rackham
diff --git a/docs/images/callouts/1.png b/docs/images/callouts/1.png
deleted file mode 100644
index 7d473430b7bec514f7de12f5769fe7c5859e8c5d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/10.png b/docs/images/callouts/10.png
deleted file mode 100644
index 997bbc8246a316e040e0804174ba260e219d7d33..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/11.png b/docs/images/callouts/11.png
deleted file mode 100644
index ce47dac3f52ac49017749a3fea53db57d006993c..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/12.png b/docs/images/callouts/12.png
deleted file mode 100644
index 31daf4e2f25b6712499ee32de9c2e3b050b691ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/13.png b/docs/images/callouts/13.png
deleted file mode 100644
index 14021a89c2ed3d4881afea6e3a315bce4f95efce..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/14.png b/docs/images/callouts/14.png
deleted file mode 100644
index 64014b75fe2e84d45ed861974c72462727979360..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/15.png b/docs/images/callouts/15.png
deleted file mode 100644
index 0d65765fcf13dcfd87914744dec8bda115e4adf1..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/2.png b/docs/images/callouts/2.png
deleted file mode 100644
index 5d09341b2f6d2ea2d1d5dad5d980f14b4b05dfd2..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/3.png b/docs/images/callouts/3.png
deleted file mode 100644
index ef7b70047158970cf4e09f1bab2954d39c2d596b..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/4.png b/docs/images/callouts/4.png
deleted file mode 100644
index adb8364eb5d21ecdd4086e16110b62ddcb42aa4a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/5.png b/docs/images/callouts/5.png
deleted file mode 100644
index 4d7eb460021e845981861d77614539314f553993..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/6.png b/docs/images/callouts/6.png
deleted file mode 100644
index 0ba694af6c07d947d219b45a629bd32c60a0f5fe..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/7.png b/docs/images/callouts/7.png
deleted file mode 100644
index 472e96f8ac36862c5645732f2fff19d06ee11a8e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/8.png b/docs/images/callouts/8.png
deleted file mode 100644
index 5e60973c213b37df93666c5a00724f34493974ae..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/callouts/9.png b/docs/images/callouts/9.png
deleted file mode 100644
index a0676d26cc2ff1de12c4ecdeefb44a0d71bc6bde..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/caution.png b/docs/images/caution.png
deleted file mode 100644
index cb9d5ea0df3fb7abe744d6ef885847c13b786d4d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/example.png b/docs/images/example.png
deleted file mode 100644
index bba1c0010ddbb148a93e8a442047ca6a68b5ef1f..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/home.png b/docs/images/home.png
deleted file mode 100644
index 37a5231bacc905886c9351a291e0d5eed68dd60e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/important.png b/docs/images/important.png
deleted file mode 100644
index 1096c232959e8cb5c2977b5eb44c1b67173cb422..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/next.png b/docs/images/next.png
deleted file mode 100644
index 64e126bdda3f2e611199744494367ad0c1a1f9ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/note.png b/docs/images/note.png
deleted file mode 100644
index 841820f7c4fb274fc45fae0a0a526a68eb31d85e..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/prev.png b/docs/images/prev.png
deleted file mode 100644
index 3e8f12fe24c6e042bc7ef4badb15449f71529ace..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/tip.png b/docs/images/tip.png
deleted file mode 100644
index a3a029d89891bff666fbdc1a082ae4a7951b5abf..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/up.png b/docs/images/up.png
deleted file mode 100644
index 2db1ce62fa575fd990108b15b0de18a64cae19ca..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/images/warning.png b/docs/images/warning.png
deleted file mode 100644
index 0b0c419df29b7e71ef14ecf5970b2abcd6d5ea01..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/index.html b/docs/index.html
deleted file mode 100644
index 0cbde07..0000000
--- a/docs/index.html
+++ /dev/null
@@ -1,19 +0,0 @@
-<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>Flume Documentation (v0.9.3-SNAPSHOT)</title><link rel="stylesheet" href="docbook.css" type="text/css"><meta name="generator" content="DocBook XSL Stylesheets V1.75.2"></head><body><div style="clear:both; margin-bottom: 4px"></div><div align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></div><span class="breadcrumbs"><div class="breadcrumbs"><span class="breadcrumb-node">Flume Documentation (v0.9.3-SNAPSHOT)</span></div></span><div lang="en" class="article" title="Flume Documentation (v0.9.3-SNAPSHOT)"><div class="titlepage"><div><div><h2 class="title"><a name="id542726"></a>Flume Documentation (v0.9.3-SNAPSHOT)</h2></div><div><div class="revhistory"><table border="1" width="100%" summary="Revision history"><tr><th align="left" valign="top" colspan="2"><b>Revision History</b></th></tr><tr><td align="left">Revision 0.9.3-SNAPSHOT</td><td align="left">November 16 2010</td></tr></table></div></div></div><hr></div><p>The following pages are the documentation for Flume v0.9.3-SNAPSHOT:</p><div class="itemizedlist"><ul class="itemizedlist" type="disc"><li class="listitem">
-<a class="ulink" href="UserGuide/index.html" target="_top">Flume User Guide</a>
-</li><li class="listitem">
-<a class="ulink" href="DeveloperGuide/index.html" target="_top">Flume Developer Guide</a>
-</li><li class="listitem">
-<a class="ulink" href="Cookbook/index.html" target="_top">Flume Cookbook</a>
-</li></ul></div></div><script type="text/javascript">
-     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-  </script><script type="text/javascript">
-     try{
-        var pageTracker = _gat._getTracker("UA-2275969-4");
-        pageTracker._setDomainName(".cloudera.com");
-        pageTracker._trackPageview();
-     } catch(err) {}
-  </script><div class="footer-text"><span align="center"><a href="index.html"><img src="images/home.png" alt="Documentation Home"></a></span><br>
-  This document was built from Flume source available at
-  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
-  </div></body></html>
diff --git a/docs/index.txt b/docs/index.txt
deleted file mode 100644
index 04ff317..0000000
--- a/docs/index.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-
-////
-   Licensed to Cloudera, Inc. under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-////
-
-Flume Documentation (v{revnumber})
-==================================
-
-The following pages are the documentation for Flume v{revnumber}:
-
-- link:UserGuide/index.html[Flume User Guide]
-- link:DeveloperGuide/index.html[Flume Developer Guide]
-- link:Cookbook/index.html[Flume Cookbook]
-
diff --git a/flume-distribution/src/main/assembly/dist.xml b/flume-distribution/src/main/assembly/dist.xml
index d3be6d1..156d9a5 100644
--- a/flume-distribution/src/main/assembly/dist.xml
+++ b/flume-distribution/src/main/assembly/dist.xml
@@ -97,7 +97,14 @@
     <!-- Copy docs directory -->
     <fileSet>
       <outputDirectory>docs</outputDirectory>
-      <directory>../docs</directory>
+      <directory>../flume-docs/target/docs</directory>
+      <includes />
+    </fileSet>
+
+    <!-- copy the man pages -->
+    <fileSet>
+      <outputDirectory>docs/man</outputDirectory>
+      <directory>../docs/man</directory>
       <includes />
     </fileSet>
 
diff --git a/flume-docs/build.xml b/flume-docs/build.xml
new file mode 100644
index 0000000..b03ac26
--- /dev/null
+++ b/flume-docs/build.xml
@@ -0,0 +1,198 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!--
+Before you can run these subtargets directly, you need
+to call at top-level: ant deploy-contrib compile-core-test
+-->
+<project name="flume" default="docs" xmlns:ivy="antlib:org.apache.ivy.ant">
+  <taskdef resource="net/sf/antcontrib/antcontrib.properties">
+    <classpath>
+      <pathelement location="${basedir}/libbuild/ant-contrib-1.0b3.jar"/>
+    </classpath>
+  </taskdef>
+  <property name="name" value="flume"/>
+  <property name="version.major" value="0"/>
+  <property name="version.minor" value="9"/>
+  <property name="version.rev" value="4"/>
+  <property name="version.numbers" value="${version.major}.${version.minor}.${version.rev}"/>
+  <property name="version" value="${version.numbers}-SNAPSHOT"/>
+  <property name="final.name" value="${name}-${version}"/>
+  <property name="year" value="2009-2011"/>
+
+  <property name="build.docs" value="${basedir}/target/docs" />
+  <property name="build.javadoc" value="${build.docs}/api" />
+
+  <property environment="env"/>
+  <property name="docs.dir" value="${basedir}/target/docs/"/>
+  <property name="src.dir" value="${basedir}/src/"/>
+  <property name="docs.src.dir" value="${src.dir}/docs"/>
+
+
+  <!-- javadoc properties  -->
+  <property name="javadoc.link.java" value="http://java.sun.com/j2se/1.6.0/docs/api/" />
+
+  <!-- version info filters set -->
+  <tstamp>
+    <format property="year" pattern="yyyy" locale="en"/>
+  </tstamp>
+
+  <!-- ================================================================== -->
+  <!-- Make Docs (UserGuide, Cookbook, DeveloperGuide)                    -->
+  <!--  converts to docbook and then to html to avoid GPL code insertion  -->
+  <!--  in ubuntu requires pkgs: asciidoc,source-highlight,graphviz       -->
+  <!-- ================================================================== -->
+  <target name="docs" depends="userguide,cookbook,developerguide,windowsguide,docindex" description="Build documentation"/>
+
+  <uptodate property="userguide.uptodate"
+	    targetfile="${build.docs}/UserGuide/index.html">
+    <srcfiles dir="${docs.src.dir}/UserGuide" includes="*"/>
+    <srcfiles dir="${docs.src.dir}/web" includes="*"/>
+  </uptodate>
+  <uptodate property="cookbook.uptodate"
+	    targetfile="${build.docs}/Cookbook/index.html" >
+    <srcfiles dir="${docs.src.dir}/cookbook" includes="*"/>
+    <srcfiles dir="${docs.src.dir}/web" includes="*"/>
+  </uptodate>
+  <uptodate property="developerguide.uptodate"
+	    targetfile="${build.docs}/DeveloperGuide/index.html" >
+    <srcfiles dir="${docs.src.dir}/DeveloperGuide" includes="*"/>
+    <srcfiles dir="${docs.src.dir}/web" includes="*"/>
+  </uptodate>
+  <uptodate property="windowsguide.uptodate"
+	    targetfile="${build.docs}/Windows/index.html" >
+    <srcfiles dir="${docs.src.dir}/Windows" includes="*"/>
+    <srcfiles dir="${docs.src.dir}/web" includes="*"/>
+  </uptodate>
+  <uptodate property="docindex.uptodate"
+	    targetfile="${build.docs}/index.html" >
+    <srcfiles dir="${docs.src.dir}/web" includes="*"/>
+  </uptodate>
+
+  <target name="userguide" description="Build UserGuide" unless="userguide.uptodate" >
+    <mkdir dir="${build.docs}/UserGuide"/>
+
+    <exec executable="asciidoc" dir="${basedir}" failonerror="true">
+      <arg line="-a revnumber=${version} -a revdate='${TODAY}' -b docbook -o ${build.docs}/UserGuide/index.xml ${docs.src.dir}/UserGuide/FullGuide"/>
+    </exec>
+
+    <exec executable="xmlto" dir="${basedir}" failonerror="true">
+      <arg line="--skip-validation -o ${build.docs}/UserGuide -m src/docs/web/html.xsl html-nochunks ${build.docs}/UserGuide/index.xml "/>
+    </exec>
+
+    <delete file="${build.docs}/UserGuide/index.xml" />
+
+    <copy todir="${docs.dir}/UserGuide">
+      <fileset dir="${docs.src.dir}/web" includes="**"/>
+    </copy>
+
+    <copy todir="${docs.dir}/UserGuide">
+      <fileset dir="${docs.src.dir}/UserGuide" includes="**/*.png"/>
+    </copy>
+  </target>
+
+  <target name="cookbook" description="Build documentation" unless="cookbook.uptodate">
+    <mkdir dir="${build.docs}/Cookbook"/>
+
+    <exec executable="asciidoc" dir="${basedir}" failonerror="true">
+      <arg line="-a revnumber=${version}  -a revdate='${TODAY}'  -b docbook -o ${build.docs}/Cookbook/index.xml ${docs.src.dir}/cookbook/Cookbook"/>
+    </exec>
+
+    <exec executable="xmlto" dir="${basedir}" failonerror="true">
+      <arg line="--skip-validation -o ${build.docs}/Cookbook -m src/docs/web/html.xsl html-nochunks ${build.docs}/Cookbook/index.xml "/>
+    </exec>
+
+    <delete file="${build.docs}/cookbook/index.xml" />
+
+    <copy todir="${docs.dir}/Cookbook">
+      <fileset dir="${docs.src.dir}/web" includes="**"/>
+    </copy>
+
+    <copy todir="${docs.dir}/Cookbook">
+      <fileset dir="${docs.src.dir}/cookbook" includes="**/*.png"/>
+    </copy>
+  </target>
+
+  <target name="developerguide" description="Build DeveloperGuide" unless="developerguide.uptodate">
+    <mkdir dir="${build.docs}/DeveloperGuide"/>
+
+    <exec executable="asciidoc" dir="${basedir}" failonerror="true">
+      <arg line="-a revnumber=${version} -a revdate='${TODAY}' -b docbook -o ${build.docs}/DeveloperGuide/index.xml ${docs.src.dir}/DeveloperGuide/FullGuide"/>
+    </exec>
+
+    <exec executable="xmlto" dir="${basedir}" failonerror="true">
+      <arg line="--skip-validation -o ${build.docs}/DeveloperGuide -m src/docs/web/html.xsl html-nochunks ${build.docs}/DeveloperGuide/index.xml "/>
+    </exec>
+
+    <delete file="${build.docs}/DeveloperGuide/index.xml" />
+
+    <copy todir="${docs.dir}/DeveloperGuide">
+      <fileset dir="${docs.src.dir}/web" includes="**"/>
+    </copy>
+
+    <copy todir="${docs.dir}/DeveloperGuide">
+      <fileset dir="${docs.src.dir}/DeveloperGuide" includes="**/*.png"/>
+    </copy>
+  </target>
+
+
+  <target name="windowsguide" description="Build Windows Guide" unless="windowsguide.uptodate">
+    <mkdir dir="${build.docs}/Windows"/>
+
+    <exec executable="asciidoc" dir="${basedir}" failonerror="true">
+      <arg line="-a revnumber=${version} -a revdate='${TODAY}' -b docbook -o ${build.docs}/Windows/index.xml ${docs.src.dir}/Windows/Windows"/>
+    </exec>
+
+    <exec executable="xmlto" dir="${basedir}" failonerror="true">
+      <arg line="--skip-validation -o ${build.docs}/Windows -m src/docs/web/html.xsl html-nochunks ${build.docs}/Windows/index.xml "/>
+    </exec>
+
+    <delete file="${build.docs}/Windows/index.xml" />
+
+    <copy todir="${docs.dir}/Windows">
+      <fileset dir="${docs.src.dir}/web" includes="**"/>
+    </copy>
+
+    <copy todir="${docs.dir}/Windows">
+      <fileset dir="${docs.src.dir}/Windows" includes="**/*.png"/>
+    </copy>
+  </target>
+
+
+  <target name="docindex" description="Build Index Page for docs" unless="docindex.uptodate">
+    <mkdir dir="${build.docs}"/>
+
+    <exec executable="asciidoc" dir="${basedir}" failonerror="true">
+      <arg line="-a revnumber=${version} -a revdate='${TODAY}' -b docbook -o ${build.docs}/index.xml ${docs.src.dir}/web/index.txt"/>
+    </exec>
+
+    <exec executable="xmlto" dir="${basedir}" failonerror="true">
+      <arg line="--skip-validation -o ${build.docs} -m src/docs/web/html.xsl html-nochunks ${build.docs}/index.xml "/>
+    </exec>
+
+    <delete file="${build.docs}/index.xml" />
+
+    <!-- there are 4 copies of this.  super inefficient -->
+    <copy todir="${docs.dir}">
+      <fileset dir="${docs.src.dir}/web" includes="**"/>
+    </copy>
+
+  </target>
+
+</project>
diff --git a/flume-docs/pom.xml b/flume-docs/pom.xml
new file mode 100644
index 0000000..4b8bad6
--- /dev/null
+++ b/flume-docs/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>flume</artifactId>
+    <groupId>com.cloudera</groupId>
+    <version>0.9.4-SNAPSHOT</version>
+  </parent>
+
+  <name>Flume Docs</name>
+  <groupId>com.cloudera</groupId>
+  <artifactId>flume-docs</artifactId>
+  <version>0.9.4-SNAPSHOT</version>
+
+  <build>
+    <plugins>
+
+	<plugin>
+          <artifactId>maven-antrun-plugin</artifactId>
+          <version>1.6</version>
+          <executions>
+            <execution>
+              <phase> package </phase>
+              <configuration>
+		<target>
+		  <property name="version">${version}</property>
+		  <ant antfile="build.xml" target="docs"/>
+		</target>
+              </configuration>
+              <goals>
+		<goal>run</goal>
+              </goals>
+            </execution>
+          </executions>
+	</plugin>
+
+    </plugins>
+  </build>
+</project>
+
diff --git a/flume-docs/src/docs/DeveloperGuide/FullGuide b/flume-docs/src/docs/DeveloperGuide/FullGuide
new file mode 100644
index 0000000..4dddeaf
--- /dev/null
+++ b/flume-docs/src/docs/DeveloperGuide/FullGuide
@@ -0,0 +1,38 @@
+Flume Developer Guide 
+=====================
+flume-dev@cloudera.org
+
+// version number comes from command line.
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+// Upgrading to use asciidoc 8.4.4
+// Build using 'asciidoc -a toc FullGuide'
+
+include::Logging[]
+include::RunningTests[]
+include::WritingDocumentation[]
+
+     ______
+    / ___//_  ______  ____
+   / /_/ / / / /    \/ __/
+  / __/ / /_/ / / / / __/
+ / / /_/\____/_/_/_/\__/
+/_/ Distributed Log Collection.
diff --git a/flume-docs/src/docs/DeveloperGuide/Logging b/flume-docs/src/docs/DeveloperGuide/Logging
new file mode 100644
index 0000000..03d60b1
--- /dev/null
+++ b/flume-docs/src/docs/DeveloperGuide/Logging
@@ -0,0 +1,51 @@
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== Logging
+
+Logging is implemented using the http://www.slf4j.org/[SLF4J API].
+
+Be sure to read the http://www.slf4j.org/docs.html[SLF4J documentation].
+
+==== Setting Up Logging
+
+Typically each class has its own logger. We declare these as follows:
+
+[source,java]
+----
+private static final Logger LOG = LoggerFactory.getLogger(MyClass.class);
+----
+
+Things to note here:
+
+* We make all loggers ++private static final++ (and in that order).
+* We name all of our loggers ++LOG++.
+* We just pass the class, we don't call ++.getName()++ on the class.
+
+==== Logging Exceptions
+
+We are using the 1.6 version of SLF4J, so we get to take advantage of
+the http://www.slf4j.org/faq.html#paramException[improved support] for
+logging exceptions.
+
+==== Logging Formatting and Performance Notes
+
+For a good discussion of this, see the
+http://www.slf4j.org/faq.html#logging_performance[relevant section]
+in the SLF4J FAQ.
+
diff --git a/flume-docs/src/docs/DeveloperGuide/RunningTests b/flume-docs/src/docs/DeveloperGuide/RunningTests
new file mode 100644
index 0000000..31ec051
--- /dev/null
+++ b/flume-docs/src/docs/DeveloperGuide/RunningTests
@@ -0,0 +1,42 @@
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== Running Tests
+
+The core suite of tests (located in src/javatest) can be run by
+using `ant test`.
+
+To run only a specific test class, you can add a special argument to
+the command line: `ant test -Dtestcase=<TestClass>` where
+`<TestClass>` is the name of a unit test class.
+
+Sometimes you may want to run all tests except for a few specific ones
+(flaky tests perhaps).  This can be done by running `ant test
+-Dtest.exclude.file=<file>`
+
+Sometimes you may want to run only a selected set of tests.  This can
+be done by running `ant test -Dtest.include.file=<file>`
+
+In both of these cases the tests are specified line by line in ant
+fileset format. The following example will recursively select all
+files that start with TestFoo.
+
+----
+**/TestFoo*
+----
+
diff --git a/flume-docs/src/docs/DeveloperGuide/WritingDocumentation b/flume-docs/src/docs/DeveloperGuide/WritingDocumentation
new file mode 100644
index 0000000..7fe5bca
--- /dev/null
+++ b/flume-docs/src/docs/DeveloperGuide/WritingDocumentation
@@ -0,0 +1,22 @@
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== Writing Documentation
+
+Documentation is written using http://www.methods.co.nz/asciidoc/[AsciiDoc].
+
diff --git a/flume-docs/src/docs/UserGuide/Advanced b/flume-docs/src/docs/UserGuide/Advanced
new file mode 100644
index 0000000..06ac257
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Advanced
@@ -0,0 +1,367 @@
+﻿////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Advanced Flume Usage
+
+This section describes in further detail, how to automate the 
+control of Flume nodes via the FlumeShell, a deep dive into Flume's dataflow 
+specification language, internals of the reliability mechanisms, how to do 
+metadata manipulations, and how to install source and sink plugins.
+
+include::FlumeShell[]
+
+=== Flume's Dataflow Specification Language
+
+Using the Flume node roles (collector, agent) is the simplest method to get up 
+and running with Flume.  Under the covers, these sources and sinks 
+(collectorSink, collectorSource, and agentSource) are actually composed of 
+primitive sinks that have been augmented with *translated components* with 
+role defaults, *special sinks* and *sink decorators*. These components make 
+configuration more flexible, but also make configuration more complicated.  
+The combination of special sinks and decorators expose a lot of details of the 
+underlying mechanisms but are a powerful and expressive way to encode rich 
+behavior.
+
+Flume enables users to enter their own composition of sinks, sources, and 
+decorators by using a domain specific data flow language.  The following 
+sections will describe this in more details.
+
+
+
+------------------------------------------------------------------------------------
+nodeName      ::=  NodeId
+simpleSource  ::=  SourceId args?
+simpleSink    ::=  SinkId args?
+decoratorSink ::=  DecoId args?
+
+source ::= simpleSource
+
+sink ::=   simpleSink            // single sink
+     |     [ sink (, sink)* ]    // fanout sink
+     |     { decoratorSink => sink } // decorator sink
+     |     decoratorSink1 decoratorSink2 ... decoratorSinkN sink // decorator sink
+     |     < sink ? sink >           // failover / choice sink
+     |     roll(...) { sink }        // roll sink
+     |     collector(...) { sink }   // generic collector sink
+
+logicalNode ::= NodeId : source | sink ;
+
+spec   ::=  (logicalNode)*
+------------------------------------------------------------------------------------
+
+
+==== Special Sinks: Fan out, Fail over, and Roll
+
+Three special sinks are FanOutSinks, FailoverSinks, and RollSinks.
+
+Fanout sinks send any incoming events to all of the sinks specified to be its 
+children.  These can be used for data replication or for processing data off 
+of the main reliable data flow path.  Fanout is similar to the Unix +tee+ 
+command or logically acts like an AND operator where the event is sent to each 
+subsink.
+
+The syntax for a FanoutSink is :
+
+----
+[ console, collectorSink ]
+----
+
+FailoverSinks are used to handle failures when appending new events. 
+FailoverSinks can be used to specify alternate collectors to contact in the 
+event the primary collector fails, or a local disk sink to store data until 
+the primary collector recovers.  Failover is similar to exception handling and 
+logically acts like an OR operator. If a failover is successful, one of the 
+subsinks has received the event.
+
+The syntax for a FailoverSink is :
+
+----
+< logicalSink("collector1") ? logicalSink("collector2") >
+----
+
+So, you could configure node "agent1" to have a failover to collector2 if 
+collector1 fails (for example, if the connection to `collector1` goes down or 
+if `collector1`'s HDFS becomes full):
+
+----
+agent1 : source | < logicalSink("collector1") ? logicalSink("collector2") 
+> ;
+----
+
+Roll sink opens and closes a new instance of its subsink every
+_millis_ milliseconds.  A roll is an atomic transition that closes
+the current instance of the sub-sink, and then opens a new instance of
+it for appending data.  A special +%{rolltag}+ attribute is available
+as an escape sequence to differentiate data in different roll periods.
+This can be used to make every roll produce a new file with a unique
+name.
+
+The syntax for a roll sink is:
+
+----
+roll(millis) sink
+----
+
+These can be composed to have even richer behavior.  For example, this sink 
+outputs to the console and has a failover collector node.
+
+----
+[ console, < logicalSink("collector1") ? logicalSink("collector2") > ]
+----
+
+This one rolls the collector every 1000 milliseconds writing to a different 
+HDFS file after each roll.
+
+----
+roll(1000) [ console, escapedCustomDfs("hdfs://namenode/flume/file-%{rolltag}") ] 
+----
+
+==== Introducing Sink Decorators
+
+Fan out and failover affect where messages go in the system but do not modify 
+the messages themselves.  To augment or filter events as they pass through the 
+dataflow, you can use *sink decorators*.
+
+Sink decorators can add properties to the sink and can modify the data streams 
+that pass through them.  For example, you can use them to increase reliability 
+via write ahead logging, increase network throughput via batching/compression, 
+sampling, benchmarking, and even lightweight analytics.
+
+The following simple sampling example uses an intervalSampler which is 
+configured to send every 10th element from source "source" to the sink "sink":
+
+----
+flumenode: source | intervalSampler(10) sink;
+----
+
+NOTE: There is an older more verbose syntax for specifiying decorators
+that wraps decorators and sinks with +{ deco => sink }+ syntax.  The
+old syntax remains valid.
+
+Here's an example that batches every 100 events together.
+
+----
+flumenode: source | batch(100) sink;
+----
+
+Like fanout and failover, decorators are also composable. Here is an example 
+that creates batches of 100 events and then compresses them before moving 
+them off to the sink:
+
+----
+flumenode: source | batch(100) gzip sink;
+----
+
+As of v0.9.3, a new more generic collector sink that better
+encapsulates the "collectorMagic" functionality.  This allows you to
+specify arbitrary sinks (or mulitple sinks) using the flume
+specification language.  So the equivalent of 
+
+----
+collectorSink("xxx","yyy-%{rolltag}", 15000)
+----
+
+is now:
+
+----
+collector(15000) { escapedCustomDfs("xxx","yyy-%{rolltag}") }
+----
+
+Here is an example of newly expressable functionality:
+
+----
+collector(15000) { [ escapedCustomDfs("xxx","yyy-%{rolltag}"), hbase("aaa", "bbb-%{rolltag}"), elasticSearch("eeee","ffff") ] }
+----
+
+This would send data to all three destinations, and only send an ack
+if all three were successful.  Combined with some recovery decorators
+(stubborn*, insistent*), this can be used to precisely express
+potentially complex failure recovery policies.
+
+==== Translations of High-level Sources and Sinks
+
+Internally, Flume translates sinks into compositions of other simpler 
+decorators, failovers, rollers to add properties.  The proper compositions 
+create pipelines that provide different levels of reliability in Flume.
+
+NOTE: This section describes how agents work, but this version of
+Flume does not expose the translations the same way the auto*Chains
+are exposed.  A future version of Flume will expose these details.
+The exact translations are still under development.
+
+Suppose you have nodes using the different agent sinks:
+
+----
+node1 : tail("foo") | agentE2ESink("bar");
+node2 : tail("foo") | agentDFOSink("bar");
+node3 : tail("foo") | agentBESink("bar");
+----
+
+In the translation phases, agentE2ESink is actually converted into these Flume 
+sinks:
+
+----
+node1 : tail("foo") | ackedWriteAhead lazyOpen stubbornAppend logicalSink("bar") ;
+node2 : tail("foo") | < lazyOpen stubbornAppend logicalSink("bar") ?  diskFailover insistentOpen lazyOpen stubbornAppend logicalSink("bar") >;
+node3 : tail("foo") | lazyOpen  stubbornAppend  logicalSink("bar");
+----
+
++ackedWriteAhead+ is actually a complicated decorator that internally
+uses rolls and some other special decorators.  This decorator
+interface allows us to manually specify wire batching and compression
+options.  For example, you could compress every 100 messages using
+gzip compression:
+
+----
+node1 : tail("foo") | ackedWriteAhead batch(100) gzip lazyOpen stubbornAppend logicalSink("bar");
+----
+
++collectorSink("xxx","yyy",15000)+ is also a bit complicated with some custom 
+decorators to handle acks.  Under the covers however, it depends on a roller 
+with a escapedCustomDfsSink inside of it.
+
+----
+roll(15000) { collectorMagic escapedCustomDfs("xxx", "yyy-%{rolltag}") }
+----
+
+Another place translations happen is with logical nodes.  Lets start of with a 
+few nodes:
+
+----
+node1 : tail("foo") | { .... => logicalSink("node2") };
+node2 : logicalSource | collectorSink("...");
+----
+
+The translation mechanisms converts logicalSources and logicalSinks into lower 
+level physical rpcSources and rpcSinks.  Lets assume that node1 is on machine 
+host1 and node2 is on machine host2.  After the translation you end up with the 
+following configurations:
+
+----
+node1 : tail("foo") | { .... => rpcSink("host2",12345) };
+node2 : rpcSource(12345) | collectorSink("..."");
+----
+
+Suppose that you swap the mapping so that node2 is now on host1 and node1 is 
+on host2.
+
+----
+# flume shell commands
+exec unmapAll
+exec map host1 node2
+exec map host2 node1
+----
+
+The original configuration will now be translated to: 
+
+----
+node1 : tail("foo") | { .... => rpcSink("host1",12345) };
+node2 : rpcSource(12345) | collectorSink("..."");
+----
+
+=== Custom Metadata Extraction
+
+While Flume can take in raw data, you can add structure to data based on the 
+nodes they flowed through, and filter out portions of data to minimize the 
+amount of raw data needed if only a portion is needed.
+
+The simplest is the +value+ decorator.  This takes two arguments: an attribute 
+name, and a value to add to the event's metadata.  This can be used to 
+annotate constant information about the source, or arbitrary data to a 
+particular node.  It could also be used to naively track the provenance of 
+data through Flume.
+
+==== Extractors
+
+Extraction operations are also available to extract values from logs that 
+have known structure.  
+
+One example is the +regex+ decorator.  This takes three arguments: a regular 
+expression, an index, and an attribute name which allows the user to use a 
+extract a particular regex group out of the body of the event and write it as 
+the value of the specified attribute.
+
+Similarly the +split+ decorator also takes three arguments: a regular
+expression, an index, and an attribute name.  This splits the body
+based on the regular expression, extracts the text group after the
+instance of the separator, and writes the value to the specified
+attribute.  One can think of this as a much simplified version of the
++awk+ Unix utility.
+
+==== Meta Data Filtering and Transformations
+
+Flume enforces an invariant that prevents the modification of an attribute 
+that has already been written upstream.  This simplifies debugging of 
+dataflows, and improves the visibility of data when debugging.
+
+If many stages are used, however, frequently a lot of extra metadata ends up 
+being moved around.  To deal with this, a few extra operations are available.
+
+A +select+ operation is available.  This operation is like SQL select, which 
+provides a relational calculus's set projection operation that modifies an 
+event so that the specified metadata fields are forwarded.
+
+// TODO Example.
+
+A +mask+ operation is also available that forwards all metadata attributes 
+except for the attributes specified.
+
+// TODO Example.
+
+A +format+ decorator is also available that uses the escape mechanisms 
+to rewrite the body of an event to a user customizable message.  This is 
+useful for outputting summary data to low volume sources.  Example: writing 
+summary information out to an IRC channel periodically.
+
+==== Role Defaults
+
+To simplify things for users, you can assign a particular role to a logical 
+node -- think of these as an "automatic" specification that have  
+default settings.  Two roles currently provided are the agent role and the 
+collector role.  Since there are many nodes with the same role, each of 
+these are called a tier.  So, for example, the agent tier consists of all the nodes 
+that are in the agent role.  Nodes that have the collector role are in the 
+collector tier.
+
+Agents and collectors roles have defaults specified in the `conf/flume-site.xml` 
+file.  Look at the `conf/flume-conf.xml` file for properties prefixed with 
+`flume.agent.*` and `flume.collector.*` for descriptions of the configuration 
+options.
+
+Each node maintains it own node state and has its own configuration. If the 
+master does not have a data flow configuration for the logical node, the 
+logical node will remain in IDLE state.  If a configuration is present, the 
+logical node will attempt to instantiate the data flow and have it work 
+concurrently with other data flows.
+
+This means that each machine essentially only lives in one tier.  In a more 
+complicated setup, it is possible to have a machine that contains many logical 
+nodes, and because each of these nodes can take on different roles, the 
+machine lives in multiple tiers.
+
+
+==== Arbitrary Data Flows and Custom Architectures
+
+With thriftSink and thriftSource, data can be sent through multiple nodes.  If
+ack injection and ack checking decorators are properly inserted, you can achieve
+reliability. 
+
+
+include::Plugins[]
+include::RateLimit[]
+
diff --git a/flume-docs/src/docs/UserGuide/Appendix b/flume-docs/src/docs/UserGuide/Appendix
new file mode 100644
index 0000000..b68eca5
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Appendix
@@ -0,0 +1,466 @@
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Appendix 
+
+=== Flume Source Catalog
+
+.Flume's Tiered Event Sources
+
+These sources and sinks are actually translated from their original 
+specification into compositions of more specific lower level configurations.  
+They generally have reasonable default arguments assigned by the properties 
+xml file or by the Master.  The defaults can be overridden by the users.
+
+[horizontal]
++collectorSource[(_port_)]+:: Collector source. Listens for data from 
+agentSinks forwarding to port +_port_+.  If port is not specified, the 
+node default collector TCP port, 35863.  This source registers itself 
+at the Master so that its failover chains can automatically be determined.
+
++autoCollectorSource+ :: Auto collector source. Creates a logical collector
+that, when assigned to a physical node, will be included in the list of
+collectors in a failover chain. This is the collector counterpart to
+auto*Chain() sinks. See the section Automatic Failover Chains for additional
+information.
+
++logicalSource+ :: Logical Source. This source has a port assigned to it by 
+the Master and listens for rpcSink formatted data.
+
+.Flume's Basic Sources
+
+These sources are untranslated and generally need all of their arguments.
+
+[horizontal]
++null+ :: Null source. Opens, closes, and returns null (last record) on next().
+
++console+ :: Stdin console source.  This is for inputting events as an
+interactive user and provides features such as edit history and
+keyboard edit shortcuts.  A flume node must be started with the +flume
+node_nowatch+ -- the watchdog does not allow console input.
+
++stdin+ :: Stdin source.  This is for piping data into a flume node's
+standard input data source.  A flume node must be started with the
++flume node_nowatch+ -- the watchdog does not allow console input.
+WARNING: although this can be used as an interactive console, it will
+hang a flume node until a newline is entered.
+
++rpcSource(_port_)+ :: A remote procedure call (RPC) server that is
+configured to listen on TCP port +_port_+. Supports both Apache-Thrift and 
+Apache-Avro RPC framework.  The type of RPC framework is specified by +event.rpc.type+ 
+property (THRIFT or AVRO), the default is THRIFT.  Note that same RPC framework is used for rpcSink.
+
++text("_filename_")+ :: One-time text file source.  One event per +\n+ 
+delimited line.
+
++tail("_filename_"[, _startFromEnd_=false]{,delim="_regex_",
+delimMode="exclude|prev|next"})+ :: Similar to Unix's tail
+utility. One line is one event. Generates events for the entire file
+then stays open for more data, and follows filename.  (e.g. if tailing
+file "foo" and then "foo" is moved to "bar" and a new file appears
+named "foo", it will finish reading the new "bar" file and then start
+from the beginning of "foo"). If the +startFromEnd+ parameter is
+false, tail will re-read from the beginning of the file.  If it is
+true, it will only start reading from the current end of file.  If the
+last line of a file does not end with a newline character ('\n'), the
++tail+ source will only send an event with this last line when the
++tail+ is closed.  See the section on tailing a file for details on
++delim+ and +delimMode+.
+
++multitail("_filename_"[, _file2_ [,_file3_ ... ] ])+ :: Like +tail+ but can 
+follow multiple files concurrently.
+
++tailDir("_dirname_"[, fileregex=".*"[, startFromEnd=false[,
+recurseDepth=0]]]{,delim="_regex_", delimMode="exclude|prev|next"})+
+:: Tails all files in a directory +dirname+ that match the specified
++fileregex+.  Be careful and make sure because the regex argument
+requires java style escaping of '\' and '\"'.  For example '\w+' would
+have to be written as "\\w+".  If a new file appears, it is added to
+the list of files to tail.  If pointed at a new directory, it will
+attempt to read all files that match! If the +startFromEnd+ parameter
+is false, tail will re-read from the beginning of each file. If it is
+true, it will only start reading from the current end of each file.
+If the +recurseDepth+ parameter is > 0 then tailDir will recurse into
+sub-directories. The value defines max level of a directory below the
++dirname+ to tail files in. Zero means do not recurse into
+sub-directories.  Note: fileregex is applied to file names only (not
+including dirs), all directories (which fit +recurseDepth+ parameter)
+are recursed into.  See the section on tailing a file for details on
++delim+ and +delimMode+.
+
++seqfile("_filename_")+ :: Read from a Hadoop sequence file formatted file, 
+with +com.cloudera.flume.handlers.hdfs.WriteableEventKey+ and 
++com.cloudera.flume.handlers.hdfs.WriteableEvent+ values. Conveniently, this 
+source can read files generated by the seqfile sink.
+
++syslogUdp(_port_)+ :: Syslog over UDP +_port_+.  This is syslog compatible.
+
++syslogTcp(_port_)+ :: Syslog over TCP +_port_+. This is syslog-ng compatible. 
+This is a server that can listen and receive on many concurrent connections.
+
++syslogTcp1(_port_)+ :: Syslog over TCP +_port_+. This is syslog-ng 
+compatible.  This is only available for a single connection and then shuts 
+down afterwards.
+
++execPeriodic("_cmdline_", _ms_)+ :: Execute an arbitrary program
+specified by +_cmdline_+.  The entire output of the execution becomes
+the body of generated messages.  +_ms_+ specifies the number of
+milliseconds to wait before the next execution (and next
+event). Ideally the program is short lived.  This does not process
+shell pipes or redirection operations -- for these write a script and
+use the script as the +_cmdline_+ argument.
+
++execStream("_cmdline_")+ :: Execute an arbitrary program specified by
++_cmdline_+.  Each line outputted will become a new event.  Ideally
+the program is long lived.  This does not process shell pipes or
+redirection operations -- for these write a script and use the script
+as the +_cmdline_+ argument.
+
+
++exec("_cmdline_"[, _aggregate_=false[, _restart_=false[,_period_=0]]])+ :: 
+Execute an arbitrary program specified by +_cmdline_+.  If the +_aggregate_+ 
+argument is +true+ entire program output is considered an event; otherwise, 
+each line is considered a new event.  If the +_restart_+ argument is +true+, 
+then the program is restarted after it exits after waiting for +_period_+ 
+milliseconds. +execStream("foo")+ is equivalent to +exec("foo", false, false, 
+0)+. +execPeriodic("foo", 1000)+ is equivalent to +exec("foo", true, true, 
+1000)+
+
++synth(_msgCount_,_msgSize_)+ :: A source that synthetically generates 
++_msgCount_+ random messages of size +_msgSize_+.  This will generate non 
+printable characters.
+
++synthrndsize(_msgCount_,_minSize_,_maxSize_)+ :: A source that synthetically 
+generates +_msgCount_+ random messages of size between randomly _minSize_ and _maxSize_.
+This will generate non printable characters.
+
++nonlsynth(_msgCount_,_msgSize_)+:: A source that synthetically generates 
++_msgCount_+ random messages of size +_msgSize_+.  This converts all `'\n'` 
+chars into `' '` chars.  This will generate non-printable characters but since 
+all randomly generated '\n' are converted, sources dependent on '\n' as a 
+record separator can get uniformly sized data.
+
++asciisynth(_msgCount_,_msgSize_)+:: A source that synthetically generates 
++_msgCount_+ random messages of size +_msgSize_+.  This converts all `'\n'` 
+chars into `' '` chars, and all non ASCII characters into printable ASCII 
+characters.
+
++twitter("_username_","_pw_"[,"_url_"])+ :: (Unsupported) A source that 
+collects data from a twitter "spritzer" stream.  +_username_+ is a twitter 
+username, +_pw_+ is the password for the user, and +_url_+ is the url for the 
+feed.  If not specified, +http://stream.twitter.com/1/statuses/sample.json+ is 
+used by default the +_url_+. See http://apiwiki.twitter.com/Streaming-API-
+Documentation for more details.
+
++irc("_server_",_port_, "_nick_","_chan_")+ :: (Unsupported) An IRC channel 
+source.  Each line sent to the channel is a new event.  It attempts to connect 
+to +_server_+ on TCP port +_port_+ (standard is 6667).  When it connects it 
+attempts to take the nickname +_nick_+, and enter channel +_chan_+ (like 
++#hadoop+   ).
+
++scribe[(+_port_+)] :: A scribe source.  This provides a network socket that 
+is compatible with data generated by Facebook's Scribe collection system.
+
++report[(periodMillis)]+ :: This source polls the local physical node for its 
+report every _periodMillis_ milliseconds and turns it into a new event.  The 
+attribute names seen from the node report page are present, and the values are 
+uninterpreted arrays of bytes.
+
+
+=== Flume Sinks Catalog
+
+.Flume's Collector Tier Event Sinks
+
+[horizontal] 
+
++collectorSink("_fsdir_","_fsfileprefix_", _rollmillis_)+ :: Collector sink.  
++_fsdir_+ is a fs directory URI such as +hdfs://namenode/path+ or +file:///
+path+.  +_fsfileprefix_+ is a file name prefix for outputted files. Both of 
+these can use escape sequences documented to bucket data as documented in the 
+*Output Bucketing* section. +_rollmillis_+ is the number of milliseconds 
+between when a HDFS file should be rolled (opened and closed).  The format for 
+data outputted by collectors is specified by the 
++flume.collector.output.format+ property.
+
+.Flume's Agent Tier Event Sinks
+
+[horizontal] 
++agentSink[("_machine_"[, _port_])]+  :: Defaults to +agentE2ESink+
+
++agentE2ESink[("_machine_"[, _port_])]+ :: Agent sink with write ahead log and 
+end-to-end ack.  Optional arguments specify a +_machine_+, and the TCP 
++_port_+ pointing to a +collectorSource+.  If none is specified, the values 
+specified by the +flume.collector.event.host+ and the +flume.collector.port+ 
+properties will be used.
+
++agentDFOSink[("_machine_"[, _port_])]+ :: DiskFailover Agent sink that stores 
+to local disk on detected failure. This sink periodically checks with the 
++_machine:port_+ and resends events if becomes alive again. Optional arguments 
+specify a +_machine_+, and the TCP +_port_+ pointing to a +collectorSource+.  
+If none is specified, the values specified by the +flume.collector.event.host+ 
+and the +flume.collector.port+ properties will be used.
+
++agentBESink[("_machine_"[, _port_])]+ :: BestEffort Agent sink.  This drops 
+messages on failures and continues sending.  Optional arguments specify a 
++_collector_+, and the TCP +_PORT_+ pointing to a +collectorSource+.  If none 
+is specified, the values specified by the +flume.collector.event.host+ and the 
++flume.collector.port+ properties will be used.
+
++agentE2EChain("_m1_[:_p1_]"[, "_m2_[:_p2_]"[,...]])+ :: Agent sink
+with write-ahead log and end-to-end ack and collector failover
+chains. +_m1:p1_+ specifies a machine and optional port of the primary
+default collector.  If all failovers are exhausted due to failures,
+and since data is already durable locally, it will back off attempts
+to send down stream.  Optional arguments specify a list of failover
+machine:port pairs in a ranked order.  If a primary collector is not
+responding, the backups are used.  The primary collectors are checked
+periodically to see if they have come back up.
+
++agentDFOChain("_m1_[:_p1_]"[, "_m2_[:_p2_]"[,...]])+ :: DiskFailover
+Agent sink that first attempts to fail over to other
+collectors. +_m1:p1_+ specifies a machine and optional port of the
+primary default collector.  If all failovers are exhausted due to
+failures, it will store to local disk.  Optional arguments specify a
+list of failover machine:port pairs in a ranked order.  If a primary
+collector is not responding, the backups are used.  The primary
+collectors are checked periodically to see if they have come back up.
+
++agentBEChain("_m1_[:_p1_]"[, "_m2_[:_p2_]"[,...]])+ :: BestEffort
+Agent sink with collector failover chains.  +_m1:p1_+ specifies a
+machine and optional port of the primary default collector.  If all
+failovers are exhausted due to failures, this drops messages. Optional
+arguments specify a +_collector_+, and the TCP +_port_+ of the
+collector.  If none is specified, the values specified by the
++flume.collector.event.host+ and the +flume.collector.port+ properties
+will be used.
+
++autoE2EChain+ :: This sink is an +agentE2EChain+ that has failover nodes 
+populated automatically by the master.
+
++autoDFOChain+ :: This sink is an +agentDFOChain+ that has failover nodes 
+populated automatically by the master.
+
++autoBEChain+ :: This sink is an +agentBEChain+ that has failover nodes 
+populated automatically by the master.
+
+.Flume's Logical Sinks
+
++logicalSink("_logicalnode_")+ :: This sink creates an rpcSink that is 
+assigned a host and IP based on the name of a logical node.  This information 
+is maintained and automatically selected by the master.
+
+.Flume's Basic Sinks
+
+[horizontal]
++null+ :: Null sink. Events are dropped
+
++console[("_formatter_")]+ :: Console sink.  Display events to process's 
+stdout using the optionally specified output formatter.
+
++text("_txtfile_"[,"_formatter_"])+ :: Textfile sink.  Write to text file 
+txtfile, using an optionally specified +_formatter_+.  If a file already 
+exists, this sink will attempt to overwrite it.
+
++seqfile("filename")+ :: Seqfile sink.  Write to a Hadoop sequence file 
+formatted file, with +com.cloudera.flume.handlers.hdfs.WriteableEventKey+ keys 
+and +com.cloudera.flume.handlers.hdfs.WriteableEvent+ values.  If a file 
+already exists, this sink will attempt to overwrite it.
+
++dfs("_hdfspath_")+ :: Hadoop dfs seqfile sink.  Write to a dfs path
+in Flume- specific Hadoop seqfile record format. The +_hdfspath_+ can
+use escape sequences to bucket data as documented in the *Output
+Bucketing* section.
+
++customdfs("_hdfspath_"[, "_format_"])+ :: Hadoop dfs formatted file
+sink.  The _hdfspath_ string is *not* escaped. The output format of
+writes to a dfs path in using specified output _format_.
+
++escapedCustomDfs("_hdfspath_", "_file_", "_format_") :: Hadoop dfs formatted 
+file sink.  The _hdfspath_ string is escaped and events will get written to 
+particular directories and filenames based on this string. The output format 
+of writes to a dfs path in using specified output _format_. The +_hdfspath_+ 
+can use escape sequences documented to bucket data as documented in the 
+*Output Bucketing* section.
+
++rpcSink("_host_"[, _port_])+ :: A remote procedure call (RPC) sink that is 
+configured to send to machine +_host_+ on TCP port +_port_+. Default port is 35861 
+and can be overridden by setting the +flume.collector.event.port+ property. 
+Supports both Apache-Thrift and Apache-Avro RPC framework.  
+The type of RPC framework is specified by +event.rpc.type+ property (THRIFT or AVRO), 
+the default is THRIFT.  Note that same RPC framework is used for rpcSource.
+
++syslogTcp("_host_"[,_port_])+ :: Syslog TCP sink.  Write to host "host" on 
+port "port" in syslog over TCP format (syslog-ng compatible). Default port is 
+TCP 514.
+
++irc("_host_",_port_, "_nick_", "_chan_")+ :: (Unsupported) An IRC channel 
+sink.  Each event is sent to the channel as a line.  It attempts to connect to 
++_server_+ on TCP port +_port_+.  When it connects it attempts to take the 
+nickname +_nick_+, and enter channel +_chan_+ (like +#hadoop+).
+
+=== Flume Sink Decorator Catalog
+
+
+.Flume's Sink Decorators
+[horizontal]
++nullDeco+ :: This is a decorator that just passes data through to its child sink.
+
++writeAhead(...)+ :: Write-ahead decorator.  Provides durability by writing 
+events to disk before sending them.  This can be used as a buffering mechanism 
+-- receive and send are decoupled in different threads.
+
++ackedWriteAhead[(_maxmillis_)]+ :: Write-ahead decorator that adds 
+acknowledgement tags and checksums to events.  Provides durability by writing 
+events to disk before sending them.  This can be used as a buffering mechanism 
+-- receive and send are decoupled in different threads.  This generates and 
+tracks groups of events, and also notifies other components to check for 
+acknowledgements.  These checks for retries are done where there is an 
+exponential backoff that can top out at _maxmillis_ milliseconds.  The default 
+value for _maxmillis_ is +flume.agent.logdir.maxage+ property.
+
++diskFailover[(_maxmillis_)]+ :: Disk failover decorator.  Events that enter 
+this decorator are sent to its sub sink.  In the event of a down stream error, 
+data is written to disk, and periodically these disk-buffered events are 
+retried.  These checks for retries are done where there is an exponential 
+backoff that can top out at _maxmillis_ milliseconds.  The default value for 
+_maxmillis_ is +flume.agent.logdir.maxage+ property.
+
++ackInjector+ :: This decorator injects an extra ack group start message on 
+open, tags appended events with an ack tag, and injects an extra ack group end 
+message.  These tags contain a checksum, for all the bodies of the events that 
+pass through the +ackInjector+.
+
++ackChecker+ :: This decorator tracks ack group start, end, and checksum 
+values inserted by +ackInjector+.  If a group has arrived and its checksum is 
+correct, it sends notifications to other components.
+
++lazyOpen+ :: This decorator tracks open/closed state of the sub sink but does 
+not actually open the sink until an append is called.  Thus if a this 
+decorator is opened and closed without any appends, the sub sink is never 
+opened.
+
++insistentOpen[(_max_[_init_[,_cumulativeMax_]],)]+ :: An insistent open 
+attempts to open its subsink multiple times until it succeeds with the 
+specified backoff properties.  This is useful for starting a network client up 
+when the network server may not yet be up.  When an attempt to open the 
+subsink fails, this exponentially backs off and then retries the open.  _max_ 
+is the max number of millis per backoff (default is Integer.MAX_VALUE).  
+_init_ is the initial number of millis to back off on the first encountered 
+failure (default 1000). _cumulativeMax_ is the maximum backoff allowed from a 
+single failure before an exception is forwarded (default is 
+Integer.MAX_VALUE).  Note that this synchronously blocks the open call until 
+the open succeeds or fails after _cumulativeMax_ millis.
+
++stubbornAppend+ :: A stubborn append normally passes through append 
+operations to its subsink.  It catches the first exception that a subsink's 
+append method triggers, and then closes, opens, and reappends to the subsink. 
+If this second attempt fails, it throws an exception. This is useful in 
+conjunction with network sinks where connections can be broken.  The open/
+close retry attempt is often sufficient to re-establish the connection.
+
++value("_attr_","_value_")+ :: The value decorator adds a new metadata 
+attribute _attr_ with the value _value_.  Agents can mark their data with 
+specific tags for later demultiplexing.
+
++mask("_attr1_"[,"_attr2_", ...])+ :: The mask decorator outputs inputted 
+events that are modified so that all metadata *except* the attributes 
+specified pass through.
+
++select("_attr1_"[,"_attr2_", ...])+ :: The select decorator outputs inputted 
+events that are modified so that *only* the metadata attributes specified pass 
+through.
+
++digest("_algorithm_","_attr_", base64="boolean")+ :: The digest decorator
+calculates a message digest of the event body and writes this value (as bytes)
+to the _attr_ attribute.  The valid algorithms are those valid for
++java.security.MessageDigest+ and includes MD5, SHA-1, SHA-256, SHA-384 and
+SHA-512. It can optionally base64 encode the digest value (defaults to false).
+
++format("_pattern_")+ :: The format decorator outputs inputted events that are 
+modified so that their bodies are replaced with an escaped version of the 
++_pattern_+ argument.  Since checksums rely on body data, this should only be 
+used on unreliable flows or reporting flows. Inappropriate use may result in 
+message loss.
+
++exDate("_attr_","_pattern_" [, "_prefix_" [, "_padding_"]])+ :: Parse a date string from 
+the _attr_ value of the event, using the _pattern_ as a reference of how to 
+parse it. The major values of the date will be assigned to new event attributes 
+using "date" as a prefix if no _prefix_ is provided. To add or remove zero padding there
+is the _padding_ variable. By default zero padding is on. Example of zero padding 2010-1-1 becomes 2010-01-01.
+More info on how to construct the _pattern_ can be found here 
+http://download-llnw.oracle.com/javase/1.4.2/docs/api/java/text/SimpleDateFormat.html
+Do note that _attr_ is an already extracted date string from the body of the event.
+exDate doesn't attempt to extract it from the body for you. Example output attributes would be
+dateday, datemonth, dateyear, datehr, datemin, datesec. Where _date_ is the _prefix_.
+
++regex("_regex_",idx,"_attr_")+ :: The regex decorator applies the regular 
+expression _regex_, extracts the _idx_ th capture group, and writes this value 
+to the _attr_ attribute.  The regex must use java-style escaping.  Thus a 
+regexs that want to use the +\d+ macro need to be specified as "\\d".
+
+regexAll("_regex_", "_name_" [, "_name_"]*)+ :: Applies the regular expression
+_regex_ to the event body, and assignes all pattern groups found to each provided 
+_name_.
+
++split("_regex_",idx,"_attr_")+ :: The split decorator uses the
+regular expression _regex_ to split the body into tokens (not
+including the splitter value). The _idx_ is then written as the value
+to the _attr_ attribute.  The regex must use java-style escaping.
+Thus, a regex that wants to use the +\d+ macro must be specified as
+"\\d".
+
++batch(_n_,_maxlatency_)+ :: buffers _n_ events and then sends one
+aggregate event.  If _maxlatency_ millis have passed, all current
+buffered events are sent out as an aggregate event.
+
++unbatch+ :: Unbatch takes an aggregate event generated by batch, splits it, 
+and then forwards its original events.  If an event is not an aggregate it is 
+just forwarded.
+
++gzip+ :: gzips a serialized event.  This is useful when used in conjunction 
+with aggregate events.
+
++gunzip+ :: gunzip's a gzip'ed event.  If the event is not a gzip event, it is 
+just forwarded.
+
++intervalSampler(_n_)+ :: Interval sampler. Every +_n_+ th event gets 
+forwarded.
+
++probSampler(_p_)+ :: Probability sampler. Every event has a probability _p_ 
+(where 0.0 &le; _p_ &le; 1.0) chance of being forwarded.
+
++reservoirSampler(_k_)+ :: Reservoir sampler.  When flushed, at most _k_ 
+events are forwarded.  If more than _k_ elements have entered this decorator, 
+exactly _k_ events are forwarded.  All events that pass through have the same 
+probability of being selected.  NOTE: This will reorder the events being sent.
+
++delay(_ms_)+ :: adds a _ms_ millisecond delay before forwarding events down 
+the pipeline.  This blocks and prevents other events from entering the 
+pipeline.  This is useful for workload simulation in conjunction with 
++asciisynth+ sources.
+
++choke[(_choke-id_)]+ :: Limits the transfer rate of data going into the sink.  
+The +choke-id+ should have been registered on the physical node where this decorator is being created using the +setChokeLimit+ command.  
+Refer to *Limiting Data Transfer Rate between Source-Sink pairs* section for more details.
+
+
+include::Environment[]
+
+=== flume-site.xml configuration settings
+
+include::Troubleshooting[]
diff --git a/flume-docs/src/docs/UserGuide/CustomInput b/flume-docs/src/docs/UserGuide/CustomInput
new file mode 100644
index 0000000..7e6102b
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/CustomInput
@@ -0,0 +1,185 @@
+﻿////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Integrating Flume with your Data Sources
+
+.WARNING 
+******************** 
+This section is incomplete.
+********************
+
+Flume's source interface is designed to be simple yet powerful and enable logging 
+of all kinds of data -- from unstructured blobs of byte, semi-structured blobs 
+with structured metadata, to completely structured data.
+
+////
+Issues: language neutrality, reliability, push vs pull, one shot vs 
+continuous. 
+////
+
+In this section we describe some of the basic mechanisms that can be used to 
+pull in data.  Generally, this approach has three flavors. *Pushing* data to 
+Flume, having Flume *polling* for data, or *embedding* Flume or Flume 
+components into an application.
+
+These mechanisms have different trade-offs -- based on the semantics of the 
+operation.
+
+Also, some sources can be *one shot* or *continuous* sources.
+
+
+=== Push Sources
+
++syslogTcp+, +syslogUdp+ :: wire-compatibility with syslog, and syslog-ng 
+logging protocols.
+
++scribe+ :: wire-compatibility with the scribe log collection system.
+
+=== Polling Sources
+
++tail+, +multitail+ :: watches a file(s) for appends.
+
++exec+ :: This is good for extracting custom data by using existing programs.
+
++poller+ :: We can gather information from Flume nodes themselves.
+
+=== Embedding Sources
+
+WARNING: these features are incomplete.
+
++log4j+
+
++simple client library+
+
+
+ // move this to gathering data from sources
+
+=== Logging via log4j Directly
+
+Flume includes specific integration support for Apache Log4j that allows end
+user applications to log directly to a Flume agent with no code modification.
+This support comes in the form of a log4j appender and can be configured in an
+application's +log4j.properties+ or +log4j.xml+ file as you would with any of
+the built-in appenders. The appender uses Flume's +avroSource()+ and converts
+each log4j +LoggingEvent+ into a Flume Avro event that can be natively handled
+by Flume.
+
+To configure log4j to log to Flume:
+
+. Ensure the proper jar files are on the application's classpath.
+. Configure the +com.cloudera.flume.log4j.appender.FlumeLog4jAvroAppender+
+  appender in the log4j configuration file.
+
+To use the Flume Avro appender, you must have the following jars on your
+application's classpath:
+
+- +flume-log4j-appender-_version_.jar+
+- +flume-core-_version_.jar+
+
+Avro jar files and its dependencies are also required.
+
+The simplest way to ensure all dependencies are properly included in your
+application's classpath is to use a build system such as Maven that handles
+transitive dependencies for you. Flume's log4j appender is available as a Maven
+project and will properly include Avro dependencies.
+
+The Flume Avro appender has a number of options users can set to affect its
+behavior. The only parameter that absolutely must be set is the port on which
+the Flume avroSource is listening. The appender assumes the Flume agent is
+running locally and that we can communicate via the hostname +localhost+. Users
+can also control the number of times to attempt reconnection before a logging
+call fails.
+
+.Parameters
+
++hostname+ :: The hostname or IP to which we should attempt to send events.
+(default: +localhost+)
+
++port+ :: The port on which Flume's avroSource is configured to listen.
+(required)
+
++reconnectAttempts+ :: The maximum number of times we should attempt to connect
+to the +avroSource()+ before throwing an exception. A setting of 0 (zero) means to
+try forever. (default: 10)
+
+.Example log4j.properties
+--------------------------------------
+log4j.debug = true
+log4j.rootLogger = INFO, flume
+
+log4j.appender.flume = com.cloudera.flume.log4j.appender.FlumeLog4jAvroAppender
+log4j.appender.flume.layout = org.apache.log4j.TTCCLayout
+log4j.appender.flume.port = 12345
+log4j.appender.flume.hostname = localhost
+log4j.appender.flume.reconnectAttempts = 10
+--------------------------------------
+
+.Example Flume configuration
+--------------------------------------
+my-app : avroSource(12345) | agentE2ESink("my-app-col", 12346)
+my-app-col : collectorSource(12346) | collectorSink("hdfs://...", "my-app-")
+--------------------------------------
+
+Note how the port referenced in the log4j.properties example matches that of
+the +avroSource()+ in the Flume configuration example.
+
+.Notes
+
+The +FlumeLog4jAvroAppender+ uses no buffering internally. This is because
+buffering would potentially create a case where, even if a Flume node is
+configured as end-to-end durable, events in the appender's internal buffer
+could be lost in the event of a failure.
+
+By setting the +reconnectAttempts+ parameter to zero (i.e. retry forever) you
+can ensure the end user application blocks should the Flume agent become
+unavailable. This is meant to satisfy users who have a zero data loss
+requirement where it's better to stop service than to not be able to log that
+it occurred.
+
+//// 
+WARNING: These instructions are out of date and currently untested.
+
+Modify hadoop-daemon.sh so that it includes Flume.
+
+Places where log4j mentioned:
+
+---- 
+bin/hadoop-daemon.sh    -- defaults to INFO,DRFA 
+conf/hadoop-env.sh      -- can be set but can cause perms issues 
+bin/hadoop              -- defaults to INFO,console 
+conf/log4j.properties   -- loggers are defined here, but the default root logger is ignored
+
+src/java/..../TaskRunner.java -- INFO,TLA 
+---- 
+////
+
+==== Example of Logging Hadoop Jobs 
+//// 
+For jobs initiated by the user, the easiest mechanism to enable Flume logging is to modify conf/hadoop-env.sh to include:
+
+export HADOOP_ROOT_LOGGER=INFO,flume,console 
+////
+
+==== Logging Hadoop Daemons 
+//// 
+To log events generated by Hadoop's daemons (tasktracker, jobtracker, datanode, secondarynamenode, namenode), modify bin/hadoop-daemon.sh so that HADOOP_ROOT_LOGGER is set to  
+
+export HADOOP_ROOT_LOGGER=INFO,flume,DRFA
+
+TODO (jon) this doesn't seem to working right now --  need to figure out why. 
+////
diff --git a/flume-docs/src/docs/UserGuide/DefaultPorts b/flume-docs/src/docs/UserGuide/DefaultPorts
new file mode 100644
index 0000000..0f190ad
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/DefaultPorts
@@ -0,0 +1,48 @@
+﻿////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+==== What are the default ports?
+
+TCP ports are used in all situations.
+
+[width="40%"] 
+|===========================================================================
+| node collector port      | +flume.collector.port+                 | 35853+
+| node status web server   | +flume.node.http.port+                 | 35862+
+| master status web server | +flume.master.http.port+               | 35871
+| master heartbeat port    | +flume.master.heartbeat.port+          | 35872
+| master admin/shell port  | +flume.master.admin.port+              | 35873
+| master gossip port       | +flume.master.gossip.port+             | 57890  
+| master report port       | +flume.report.server.port+             | 45678
+| master -> zk port        | +flume.master.zk.client.port+          |  3181
+| zk -> zk quorum port     | +flume.master.zk.server.quorum.port+   |  3182
+| zk -> zk election port   | +flume.master.zk.server.election.port+ |  3183
+|===========================================================================
+
+When using auto chains for collectors, each collector source will
+automatically search for a free port by incrementing starting from
+port +flume.collector.port+ and incrementing the port number.   
+
+When there is more than one physical node on a machine, the node
+status web server will attempt to bind to port +flume.node.http.port+.
+If the bind fails, the port number will be incremented and the bind
+retried until a free port is found.  The port that is eventually bound
+to is logged.
+
+
diff --git a/flume-docs/src/docs/UserGuide/Environment b/flume-docs/src/docs/UserGuide/Environment
new file mode 100644
index 0000000..93f83fa
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Environment
@@ -0,0 +1,59 @@
+
+=== Flume Environment Variables
+
+This section describes several environment variables that affect how
+Flume operates.  The +flume+ script in +./bin+ uses these system
+environment variables.  Many of these variables are set by the
++flume-daemon.sh+ script used when flume is run as a daemon.
+
+[horizontal]
++FLUME_PID_DIR+ :: The directory where a flume node or flume
+master will drop pid files corresponding to the daemon process.
+
++FLUME_CLASSPATH+ :: The custom java classpath environment variable
+additions you want flume to run with.  These values are prepended to
+the normal Flume generated CLASSPATH.  WARNING: The +flume+ script
+overwrites the standard CLASSPATH when it is executed.
+
++FLUME_LOG_DIR+ :: The directory where debugging logs
+generated by the flume node or flume master are written.
+
++FLUME_LOGFILE+ :: This sets the suffix for the logfiles generated by
+flume node or flume master.
+
++FLUME_ROOT_LOGGER+ :: The log4j logging setting for the executing
+command.  By default it is "INFO,console".
+
++ZOOKEEPER_ROOT_LOGGER+ :: The log4j logging setting for a master's
+embedded zookeeper server's logs.  By default it is "ERROR,console".
+
++WATCHDOOG_ROOT_LOGGER+ :: The log4j logging setting for the logging
+the watchdog that wraps flume nodes and flume masters generates.  By
+default it is "INFO,console".
+
++FLUME_CONF_DIR+ :: The directory where the +flume-site.xml+ and
++flume-conf.xml+ files flume node and flume master will use reside.
+This defaults to +./conf+ if a +./conf/flume-conf.xml+ file is found,
+or to +/etc/flume/conf/+ if it +flume-conf.xml+ is found there.
+
++HADOOP_HOME+ :: The directory where Hadoop jars are expected to be
+found.  If not specified it will use jars found in +/usr/lib/hadoop+
+or +./lib/+.
+
++FLUME_DEVMODE+ :: If this value is set to "true" the +./libbuild+
+jars which include ant jars required to compile JSP servlets will be
+included in the CLASSPATH.
+
++FLUME_VERBOSE+ :: If it this is toggled, the flume script will print
+out the command line being executed.
+
++FLUME_VERBOSE_JAVA+ :: If this is toggled along with FLUME_VERBOSE,
+the "-verbose" flag will be passed to the JVM running flume.
+
++LD_LIBRARY_PATH+ :: Populate with a : separated list of file paths to
+extend the java.library.path aspect of Flume. This allows you to include 
+native libraries in the java lookup path. Example usage would be with Lzo
+where you need to extend the path with the Lzo C libraries contained in your
+Hadoop install.
+
+
diff --git a/flume-docs/src/docs/UserGuide/FlumeShell b/flume-docs/src/docs/UserGuide/FlumeShell
new file mode 100644
index 0000000..fc5f223
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/FlumeShell
@@ -0,0 +1,208 @@
+﻿////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== The Flume Command Shell
+
+So far, you have been modifying the state of Flume using a simple (but 
+primitive) web interface to a Master server.  
+
+Flume also provides a shell, which allows the user to type commands into a 
+terminal and have them executed on a Flume deployment.
+
+All of the commands available in the web form are available in the
+Flume Shell.  The Flume Shell, however, actually has extra controls
+for command submission control and state checking that aid
+scriptability.
+
+==== Using the Flume Command Shell
+
+You can start the FlumeShell by running +flume shell+ in a terminal window. 
+The +connect+ command can be used to establish a connection to any Master 
+server.
+
+----
+hostname:~/flume$ flume shell
+[flume (disconnected)] connect localhost:35873
+Connecting to Flume master localhost:35873...
+[flume localhost:35873]
+----
+
+----
+hostname:~/flume$ flume shell -c localhost:35873
+Connecting to Flume master localhost:35873...
+[flume localhost:35873]
+----
+
+******
+The port to use is the value of +flume.config.admin.port+ and defaults to 35873. 
+******
+
+The command line parameters for the Flume Shell are as follows:
+
+----
+usage: FlumeShell [-c <arg>] [-e <arg>] [-q] [-s <arg>]
+ -?         Command line usage help
+ -c <arg>   Connect to master:port
+ -e <arg>   Run a single command
+ -q         Run in quiet mode - only print command results
+ -s <arg>   Run a FlumeShell script
+----
+
+The FlumeShell makes scripting Flume possible - either by using a single 
+invocation with +-e+ or by running a script of commands with +-s+. It is also 
+possible to pipe +stdin+ to the FlumeShell as in the following example:
+
+----
+echo "connect localhost:35873\ngetconfigs\nquit" | flume shell -q
+----
+
+.Flume Commands
+
+You can press Tab any time for some hints on available commands.  If you start 
+typing a command you can use TAB to complete command.
+
++help+ :: List the commands available in the shell.
+
++connect _master:port_+ :: connect to a master at machine _master_ on port 
+_port_.
+
++config+ _logicalnode_ _source_ _sink_ :: configure a single logical node 
+_logicalnode_ with source _source_ and sink _sink_.  _source_ and _sink_ will 
+likely need quotes to support some of the Flume configuration syntax.
+
++getnodestatus+ :: Output the status of the nodes the master knows
+about. Nodes are in either HELLO, CONFIGURING, ACTIVE, IDLE, ERROR,
+DECOMMISSIONED, or LOST states.  When a node shows up initially it is
+HELLO state.  When a node is being configured, it is in CONFIGURING
+state.  Once events are being pumped from source to sink, the node is
+in ACTIVE state.  If a node has drained its source (and the source is
+not "endless") it will enter IDLE state.  If a node encountered an
+unrecoverable error or exited without flushing, it will be in ERROR
+state.  A node is DECOMMISSIONED if it is removed on the master, and
+LOST if it has not been seen by the master for a "long time".
+
++getconfigs+ :: This gets and dumps the configuration specifications of all 
+the logical nodes the master knows about.
+
++getmappings [_physical node_]+ :: Display all logical nodes mapped to
+_physical node_ or all mappings if _physical node_ is omitted.
+
++exec+ :: Synchronously execute a command on the master.  This command will 
+block until it is completed.
+
++source _file_+ :: Reads the specified file and attempts to execute all of the 
+specified commands.
+
++submit+ :: Asynchronously execute a command on the master.  This command will 
+return immediately and allows the submission of other commands.  The command 
+ID of the last command submitted is recorded.
+
++wait _ms_ [_cmdid_]+ :: This commands blocks for up to +ms+ milliseconds 
+until +cmdid+ has entered the SUCCEEDED or FAILED state. If +ms+ is 0 the 
+command may block forever.  If the command times out, the shell will 
+disconnect.  This is useful in conjunction with +submitted+ commands.
+
++waitForNodesActive _ms_ node1 [node2 [...]]+ :: This command blocks for up to 
++ms+ milliseconds until the specified list of nodes have entered the ACTIVE or 
+CONFIGURING state.  If ms==0 then the command may block forever.
+
++waitForNodesDone _ms_ node1 [node2 [...]]+ :: This command blocks for up to 
++ms+ milliseconds until the specified list of nodes have entered the IDLE, 
+ERROR, or LOST state.
+
++quit+ :: Exit the shell.
+
+.Exec and Submit commands
+
+Both the web form and the FlumeShell are interfaces to the same command 
+processing infrastructure inside Flume. This section introduces the 
+FlumeShell and show how you can use it to make administering Flume more 
+simple.
+
+These commands are issued and run as if run from the master.  In the command 
+shell they have the form:
+
++exec _command_ [_arg1 [_arg2_ [ ... ] ] ]+
+
++submit _command_ [_arg1 [_arg2_ [ ... ] ] ]+
+
+Complex arguments like those with spaces, or non alpha-numeric characters can 
+be expressed by using "double quotes"s and `single quotes's.  If enclosed in 
+double quotes, the bodies of the strings are Java string unescaped.  If they 
+are enclosed in single quotes, arbitrary characters can be included except for 
+the ' character.
+
++exec+ commands block until they are completed.  +submit+ commands are 
+asynchronously sent to the master in order to be executed.  +wait+ are 
+essentially joins for recently +submit+ ted commands.
+
++noop+ :: This command contacts the master and issues a noop (no
+operation) command.
+
++config _logicalnode_ 'source' 'sink'+ :: This command configures a node.  
+This is nearly identical to the 'config' command.
+
++multiconfig '_flumespec_'+ :: This command configures a set of nodes on the 
+master using the aggregated format.
+
++unconfig _logicalnode_+ :: This command changes the configuration of a 
+particular node to have a +null+ source and a +null+ state.  +refresh 
+_logicalnode_+ :: This command refreshes the current configuration of a 
+logical node.  This forces the logicalnode to stop and then restart.  This 
+also causes a master re-evaluation that may change the failover lists.
+
++refreshAll _logicalnode_+ :: This atomically issues a refresh command to all 
+of the logical nodes.
+
++save '_filename_'+ :: This saves the current configuration to the master's 
+disk.
+
++load '_filename_'+ :: This augments the current configuration with the 
+logical node specifications found in +_filename_+.
+
++map _physicalnode_ _logicalnode_+:: This creates a new mapping between logical
+node _logicalnode_ and physical node +_physicalnode_+.  The node starts with a
++null+ source and a +null+ sink, and updates its configuration specified at the
+master when it begins heartbeating.  Thus if a logical node configuration
+already exists and is mapped, it will pick up the configuration for the logical
+node.
+
++spawn _physicalnode_ _logicalnode_+:: The +spawn: command is a synonym for the
++map+ command and has been deprecated.
+
++decommission _logicalnode_+ :: This removes a logical node from the logical 
+node configuration table, and unmaps it from any physical nodes it may be 
+installed on.
+
++unmap _physicalnode_ _logicalnode_+ :: This command breaks the assignment of 
+a _logicalnode_ from machine _physicalnode_.  A logical node can be reassigned 
+to another physical node using the +map+ command.
+
++unmapAll+ :: This command breaks the assignment of all logical node from 
+physical nodes.  A logical node can be reassigned to another physical node 
+using the +map+ command.
+
++purge+ _logicalnode_ :: This command removes an entry from the
+logical node status table.  It is useful for removing DECOMMISSIONED
+or LOST nodes.
+
++purgeAll+ :: This command removes *all* entries from the logical node
+status table.  It is useful for removing DECOMMISSIONED or LOST nodes.
+Note that even nodes that are in ACTIVE/IDLE/ERROR states will be
+removed but added again after their next heartbeat.
+
diff --git a/flume-docs/src/docs/UserGuide/FullGuide b/flume-docs/src/docs/UserGuide/FullGuide
new file mode 100644
index 0000000..f2a248a
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/FullGuide
@@ -0,0 +1,47 @@
+Flume User Guide 
+=================
+flume-dev@cloudera.org
+
+// version number comes from command line.
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+// Upgrading to use asciidoc 8.4.4
+// Build using 'asciidoc -a toc FullGuide'
+
+include::Introduction[]
+// include::Installation[]  // In confluence Wiki
+include::QuickStart[]
+include::PseudoDist[]
+include::FullyDist[]
+include::CustomInput[]
+include::UsingOutput[]
+include::Advanced[]
+include::Security[]
+// include::Experimental[] // in really bad shape
+include::Appendix[]
+include::Glossary[]
+include::VersionHistory[]
+     ______
+    / ___//_  ______  ____
+   / /_/ / / / /    \/ __/
+  / __/ / /_/ / / / / __/
+ / / /_/\____/_/_/_/\__/
+/_/ Distributed Log Collection.
diff --git a/flume-docs/src/docs/UserGuide/FullyDist b/flume-docs/src/docs/UserGuide/FullyDist
new file mode 100644
index 0000000..aebaefc
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/FullyDist
@@ -0,0 +1,711 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Fully-distributed Mode
+
+The main goal for Flume is to collect logs and data from many
+different hosts and to scale and intelligently handle different
+cluster and network topologies.
+
+To deploy Flume on your cluster, do the following steps.
+
+.Steps to Deploy Flume On a Cluster 
+* Install Flume on each machine. 
+* Select one or more nodes to be the Master. 
+* Modify a static configuration file to use site specific properties. 
+* Start the Flume Master node on at least *one* machine. 
+* Start a Flume node on *each* machine.
+
+The following section describes how to manually configure the
+properties file to specify the Master for each node, and how to set
+default values for parameters.  Sections afterwards describe a data
+flow configuration for a larger system, how to add more capacity by
+adding collectors, and how to improve the reliability of the Master by
+adding multiple Masters.
+
+=== Static Configuration Files
+
+In the previous sections, you used Flume on a single machine with the
+default configuration settings. With the default settings, nodes
+automatically search for a Master on `localhost` on a standard
+port. In order for the Flume nodes to find the Master in a fully
+distributed setup, you must specify site-specific static configuration
+settings.
+
+Site-specific settings for Flume nodes and Masters are configured by
+properties in the `conf/flume-site.xml` file found on each machine.
+If this file is not present, the commands default to the settings
+found in `conf/flume-conf.xml`.  In the following example, you set up
+the property that points a Flume node to search for its Master at a
+machine called `master`.
+
+// xml source code highlighting -- requires source-highlight library. 
+.`conf/flume-site.xml` 
+[source,xml] 
+------------------------------------------------------------------------------------ 
+<?xml version="1.0"?> 
+<?xml-stylesheet type="text/xsl"  href="configuration.xsl"?>
+
+<configuration>   
+<property>     
+<name>flume.master.servers</name>     
+<value>master</value>   
+</property> 
+</configuration> 
+------------------------------------------------------------------------------------
+
+
+==== Using Default Values
+
+When you are using agent/collector roles, you can add the following
+configuration properties to your +flume-site.xml+ file to set up the
+default hosts used as collector.
+ 
+[source,xml] 
+------------------------------------------------------------------------------------ 
+...   
+<property>     
+<name>flume.collector.event.host</name>     
+<value>collector</value>     
+<description>This is the host name of the default "remote"     collector.      
+</description>   
+</property>
+  <property>     
+<name>flume.collector.port</name>     
+<value>35853</value>     
+<description>This default tcp port that the collector listens to     in order to receive events it is collecting.     
+</description>   
+</property>
+
+... 
+------------------------------------------------------------------------------------
+
+This will make the `agentSink` with no arguments default to using
+`flume.collector.event.host` and `flume.collector.port` for their
+default target and port.
+
+In the following example, a larger setup with several agents push data
+to a collector. There are seven Flume nodes -- six in the agent tier,
+and one in the collector tier.
+
+["graphviz", "singleCollector.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+node [shape=record];   
+"agent A" -> collector -> HDFS;   
+"agent B" -> collector;   
+"agent C" -> collector;   
+"agent D" -> collector;   
+"agent E" -> collector;   
+"agent F" -> collector;
+
+  node [shape=none]   
+"agent tier" -> "collector tier" -> "storage tier"
+
+ } 
+---------------------------------------------------------------------
+
+An explicit configuration fills in all of the parameters:
+
+---- 
+agentA : src | agentSink("collector",35853); 
+agentB : src | agentSink("collector",35853); 
+agentC : src | agentSink("collector",35853); 
+agentD : src | agentSink("collector",35853); 
+agentE : src | agentSink("collector",35853); 
+agentF : src | agentSink("collector",35853); 
+collector : collectorSource(35853) | collectorSink("hdfs://namenode/flume/","srcdata"); 
+----
+
+NOTE: When specifying destinations for agentSinks, use the _hostname
+and port of the target machine_.  The default name for a node is its
+hostname.  However, if there are multiple logical nodes, you must use
+the machine's host name, _not the name of the logical node_. In the
+preceding examples, agent[A-F] and collector are the _physical host
+names_ of the machines where these configurations are running.
+
+You can rely on the default ports set in the configuration files:
+
+---- 
+agentA : src | agentSink("collector"); 
+agentB : src | agentSink("collector"); 
+agentC : src | agentSink("collector"); 
+agentD : src | agentSink("collector"); 
+agentE : src | agentSink("collector"); 
+agentF : src | agentSink("collector"); 
+collector : collectorSource | collectorSink("hdfs://namenode/flume/","srcdata");
+----
+
+You can rely on the default ports and default collector host:
+
+---- 
+agentA : src | agentSink 
+agentB : src | agentSink 
+agentC : src | agentSink 
+agentD : src | agentSink 
+agentE : src | agentSink 
+agentF : src | agentSink 
+collector : collectorSource | collectorSink("hdfs://namenode/flume/","srcdata");
+----
+
+WARNING: Using defaults can make writing data flow configurations more concise, but may obscure the details about how different nodes are connected to each other.
+
+.Reliability Modes 
+**************** 
+You can tune the reliability level of the agents, simply by specifying a different kind of 
+agent sink.  There are three levels available, and three corresponding agents:
+
++agentE2ESink[("_machine_"[,_port_])]+ :: End to end.  This version uses the WAL, relies on an acknowledgement, and will retry if no acknowledgement is received.
+
++agentDFOSink[("_machine_"[,_port_])]+ :: Disk Failover (store on failure).  This agent writes to disk only if it can detect a failure on the collector.  If any data is stored to disk, it periodically retires the network connection and attempts to resend.
+
++agentBESink[("_machine_"[,_port_])]+ :: Best Effort.  This agent does not write to disk at all, and drops messages in the event of collector failures.
+
+The previous examples use the +agentSink+.  This is an alias for the +agentE2ESink+. 
+****************
+
+
+
+=== Multiple Collectors
+
+Having multiple collectors can increase the log collection throughput
+and can improve the timeliness of event delivery by increasing
+collector availability.  Data collection is parallelizable; thus, load
+from many agents can be shared across many several collectors.
+
+
+==== Partitioning Agents across Multiple Collectors
+
+The preceding graph and dataflow spec shows a typical topology for
+Flume nodes.  For reliable delivery, in the event that the collector
+stops operating or disconnects from the agents, the agents would need
+to store their events to their respective disks locally.  The agents
+would then periodically attempt to recontact a collector.  Because the
+collector is down, any analysis or processing downstream is blocked.
+
+["graphviz", "multiCollector.png"] 
+--------------------------------------------------------------------- 
+digraph multi_collector {
+   rankdir=LR;  
+
+  node [shape=record];   
+collector [label="<A> collector A | <B> collector B | <C> collector C"];
+
+  collector:A -> HDFS;   
+collector:B -> HDFS;   
+collector:C -> HDFS;   
+"agent A" -> collector:A;   
+"agent B" -> collector:A;   
+"agent C" -> collector:B;   
+"agent D" -> collector:B;   
+"agent E" -> collector:C;   
+"agent F" -> collector:C;
+
+
+   node [shape=none]   
+"agent tier" -> "collector tier" -> "storage tier"
+
+} 
+---------------------------------------------------------------------
+
+When you have multiple collectors as in the preceding graph and
+dataflow spec, downstream progress is still made even in the face of a
+collector's failure.  If collector B goes down, agent A, agent B,
+agent E, and agent F continue to deliver events via collector A and
+collector C respectively.  Agent C and agent D may have to queue their
+logs until collector B (or its replacement) comes back online.
+
+The following configuration partitions the work from the set of agents
+across many collectors.  In this example, each of the collectors
+specify the same DFS output directory and file prefixes, aggregating
+all of the logs into the same directory.
+
+---- 
+agentA : src | agentE2ESink("collectorA",35853); 
+agentB : src | agentE2ESink("collectorA",35853); 
+agentC : src | agentE2ESink("collectorB",35853); 
+agentD : src | agentE2ESink("collectorB",35853); 
+agentE : src | agentE2ESink("collectorC",35853); 
+agentF : src | agentE2ESink("collectorC",35853); 
+collectorA : collectorSource(35853) | collectorSink("hdfs://...","src"); 
+collectorB : collectorSource(35853) | collectorSink("hdfs://...","src"); 
+collectorC : collectorSource(35853) | collectorSink("hdfs://...","src"); 
+----
+
+==== Manually Specifying Failover Chains
+
+
+["graphviz", "failoverCollector.png"] 
+--------------------------------------------------------------------- 
+digraph multi_collector {   
+rankdir=LR;  
+
+  node [shape=record];   
+collector [label="<A> collector A | <B> collector B | <C> collector C"];  
+
+  collector:A -> HDFS;   
+collector:B -> HDFS [ style="dotted" label = "down" ];  
+collector:C -> HDFS;   
+"agent A" -> collector:A;   
+"agent B" -> collector:A;   
+"agent C" -> collector:B  [ style="dotted" label = "down" ];   "agent D" -> collector:B  [ style="dotted" label = "down" ];   "agent C" -> collector:A;   
+"agent D" -> collector:C;   
+"agent E" -> collector:C;   
+"agent F" -> collector:C;
+
+  node [shape=none];   
+"agent tier" -> "collector tier" -> "storage tier"
+
+} 
+---------------------------------------------------------------------
+
+When you have multiple collectors writing to the same storage
+location, instead of having agent C and agent D queue indefinitely,
+you can instead have them fail over to other collectors.  In this
+scenario, you could have agent C and agent D fail over to collector A
+and collector C respectively, while periodically checking to see if
+collector B has returned.
+
+To specify these setups, use agents with *failover chains*. Similarly
+to single collector agents, there are three levels of reliability for
+the failover chain agents: +agentE2EChain+, +agentDFOChain+, and
++agentBEChain+.
+
+In the following example, you manually specify the failover chain
+using +agentE2EChain+, an agent with end-to-end reliability with
+multiple failover collectors.  +agentA+ in this situation will
+initially attempt to send to +collectorA+ on port +35853+.  The second
+argument in +agentA+ 's sink specifies the collector to fall back onto
+if the first fails.  You can specify an arbitrary number of
+collectors, but you must specify at least one.
+
+---- 
+agentA : src | agentE2EChain("collectorA:35853","collectorB:35853"); 
+agentB : src | agentE2EChain("collectorA:35853","collectorC:35853"); 
+agentC : src | agentE2EChain("collectorB:35853","collectorA:35853"); 
+agentD : src | agentE2EChain("collectorB:35853","collectorC:35853"); 
+agentE : src | agentE2EChain("collectorC:35853","collectorA:35853"); 
+agentF : src | agentE2EChain("collectorC:35853","collectorB:35853"); 
+collectorA : collectorSource(35853) | collectorSink("hdfs://...","src"); 
+collectorB : collectorSource(35853) | collectorSink("hdfs://...","src"); 
+collectorC : collectorSource(35853) | collectorSink("hdfs://...","src"); 
+----
+
+NOTE: In this section, +agent[A-F]+ and +collector[A-B]+ are physical host names.
+
+As in the single collector case, if no port number is specified, the agent defaults to using the +flume.collector.port+.
+
+---- 
+agentA : src | agentE2EChain("collectorA","collectorB"); 
+agentB : src | agentE2EChain("collectorA","collectorC"); 
+agentC : src | agentE2EChain("collectorB","collectorA"); 
+agentD : src | agentE2EChain("collectorB","collectorC"); 
+agentE : src | agentE2EChain("collectorC","collectorA"); 
+agentF : src | agentE2EChain("collectorC","collectorB"); collectorA : collectorSource | collectorSink("hdfs://...","src"); 
+collectorB : collectorSource | collectorSink("hdfs://...","src"); 
+collectorC : collectorSource | collectorSink("hdfs://...","src"); 
+----
+
+.Reliability and failover chain semantics 
+****************
+
+* The +agentE2EChain+ always _writes to WAL first_ and then attempts
+  to send to the different collectors.  In the event of a collector
+  failure, it fails over to another collector.
+
+* The +agentDFOChain+ always attempts to send data to the different
+  collectors, and _only writes to disk if all collectors fail_.
+
+* The +agentBEChain+ attempts to send message to each collector in
+  succession in the event of failures.  If all collectors fail, it
+  drops messages.
+
+****************
+
+
+==== Automatic Failover Chains
+
+WARNING: The automatic failover chain feature does not currently work
+when using multiple masters.
+
+Flume also provides a mechanism that automatically assigns failover chains based on how nodes are configured.  As collector nodes are assigned in the Flume Master, the Master attempts to distribute the agents evenly amongst the collectors.  In the face of failure, each agent is assigned a different failover chain.  This mitigates the chances of another collector becoming overloaded in the event of failure of a collector.
+
+To specify a node to use the failover chains, use either the +autoE2EChain+, +autoDFOChain+, or +autoBEChain+ agent sink.  Because the Master calculates the failover chains, these sinks take no explicit arguments.  
+
+---- 
+agentA : src | autoE2EChain ; 
+agentB : src | autoE2EChain ; 
+agentC : src | autoE2EChain ; 
+agentD : src | autoE2EChain ; 
+agentE : src | autoE2EChain ; 
+agentF : src | autoE2EChain ; 
+collectorA : autoCollectorSource | collectorSink("hdfs://...", "src"); 
+collectorB : autoCollectorSource | collectorSink("hdfs://...", "src"); 
+collectorC : autoCollectorSource | collectorSink("hdfs://...", "src"); 
+----
+
+The Master updates the configuration of the agents based on the current collectors in the system.  When new collectors are added to the system, the Master updates the failover chains of agents to rebalance.
+
+NOTE: If the Master has no nodes with +autoCollectorSource+ as its source, the agent's automatic chains will report a +fail("...")+ chain which will wait for +autoCollectorSource+ s to be specified.  If the nodes are not mapped, they will report a different +fail+ sink notifying you that the node is unmapped (isn't associated with a host/port).
+
+TIP: You can see the translation of the auto*Chain configuration in the node configuration table under the translated configuration column.  This is a little declarative specification of the failure recovery behavior of the sink.  More details on this are in the Advanced section of this guide, and in future revisions the translations for the agents and other chains will also be presented.
+
+include::LogicalNodes[]
+
+=== Multiple Masters
+
+WARNING: The automatic failover chains, automatic flow isolation, and
+logical source/sink feature does not currently work when using
+multiple masters.
+
+
+The Master has two main jobs to perform. The first is to keep track of all the nodes in a Flume deployment and to keep them informed of any changes to their configuration. The second is to track acknowledgements from the end of a Flume flow that is operating in *reliable mode* so that the source at the top of that flow knows when to stop transmitting an event.
+
+Both these jobs are critical to the operation of a Flume deployment. Therefore, it is ill-advised to have the Master live on a single machine, as this represents a single point of failure for the whole Flume service (see 'failure modes' for more detail).
+
+Flume therefore supports the notion of multiple Masters which run on physically separate nodes and co-ordinate amongst themselves to stay synchronized. If a single Master should fail, the other Masters can take over its duties and keep all live flows functioning. This all happens transparently with a little effort at configuration time. Nodes will automatically fail over to a working Master when they lose contact with their current Master. 
+
+==== Standalone Mode Compared to Distributed Mode
+
+The Flume Master can be run in one of two ways. 
+
+- *Standalone* mode - this is where the Master runs on a single machine. This is easy to administer, and simple to set-up, but has disadvantages when it comes to scalability and fault-tolerance.
+
+- *Distributed* mode - this is where the Master is configured to run on several machines - usually three or five. This option scales to serve many Flows, and also has good fault-tolerance properties.
+
+Large production deployments of Flume should run a distributed Master so that inevitable machine failures do not impact the availability of Flume itself. For small deployments the issue is less clear-cut - a distributed Master means reserving more computing resources that could be used instead for nodes or other services, and it is possible to recover from many failure modes in a timely manner with human intervention. The choice between distributed and standalone Masters is ultimately dependent both on your use case and your operating requirements.
+
+==== Running in Standalone Mode
+
+********************************* 
+Standalone mode is where Flume has only one Master node. The configuration described below needs to be done on that machine only. 
+*********************************
+
+Whether the Flume Master starts in distributed or standalone mode is indirectly controlled by how many machines are configured to run as Master servers. To run in standalone mode, a single configuration property +flume.master.servers+ must be set:
+
+--------------------------------------- 
+<property>   
+<name>flume.master.servers</name>   
+<value>hostA</value> 
+</property> 
+---------------------------------------
+
+The value of +flume.master.servers+ is a comma-separated list of all the machine names (or IP addresses) that will be Master servers. If this list contains only machine name, the Flume Master will start in standalone mode. If there is more than one machine name in the list, the Flume Master will start in distributed mode.
+
+There's no other configuration required for standalone mode. Flume will use reasonable default values for any other master-related variables. To start the Master, from the command prompt type:
+
+----------------------- 
+$ flume master 
+-----------------------
+
+from +$FLUME_HOME+. A number of log messages should print to the screen. After the server is running, you can check that everything is working properly by visiting the web interface at +http://master-node-ip:35871/+, 
+where +master-node-ip+ is the IP address (or hostname) of the Master node. If you see a web page, the Master is running.
+
+==== Running in Distributed Mode
+
+************************** 
+Distributed mode runs the Flume Master on several machines. Therefore the configuration described below should be done on *every* Master machine, except where noted. **************************
+
+Running the Flume Master in distributed mode provides better fault tolerance than in standalone mode, and scalability for hundreds of nodes.
+
+Configuring machines to run as part of a distributed Flume Master is nearly as simple as standalone mode. As before, +flume.master.servers+ needs to be set, this time to a list of machines:
+
+--------------------------------------- 
+<property>   
+<name>flume.master.servers</name>   
+<value>masterA,masterB,masterC</value> 
+</property> 
+---------------------------------------
+
+.How many machines do I need? 
+****************************** 
+The distributed Flume Master will continue to work correctly as long as more than half the physical machines running it are still working and haven't crashed. Therefore if you want to survive one fault, you need three machines (because 3-1 = 2 > 3/2). For every extra fault you want to tolerate, add another two machines, so for two faults you need five machines. Note that having an even number of machines doesn't make the Flume Master any more fault-tolerant - four machines only tolerate one failure, because if two were to fail only two would be left functioning, which is not more than half of four. Common deployments should be well served by three or five machines.
+******************************
+
+The final property to set is *not* the same on every machine - every node in the Flume Master must have a unique value for +flume.master.serverid+.
+
+.Note 
+***************************** 
++flume.master.serverid+ is the only Flume Master property that _must_ be different on every machine in the ensemble. *****************************
+
+.masterA 
+[source,xml] 
+-------------------------------------- 
+<property>   
+<name>flume.master.serverid</name>   
+<value>0</value> 
+</property> 
+--------------------------------------
+
+.masterB 
+[source,xml] 
+-------------------------------------- 
+<property>   
+<name>flume.master.serverid</name>   
+<value>1</value> 
+</property> 
+--------------------------------------
+
+.masterC 
+[source,xml] 
+-------------------------------------- 
+<property>   
+<name>flume.master.serverid</name>   
+<value>2</value> 
+</property> 
+--------------------------------------
+
+
+The value for +flume.master.serverid+ for each node is the index of
+that node's hostname in the list in +flume.master.ensemble+, starting
+at 0. For example +masterB+ has index 1 in that list. The purpose of
+this property is to allow each node to uniquely identify itself to the
+other nodes in the Flume Master.
+
+This is all the configuration required to start a three-node
+distributed Flume Master. To test this out, we can start the Master
+process on all three machines:
+
+-------------------------------- 
+[flume@masterA] flume master
+
+[flume@masterB] flume master
+
+[flume@masterC] flume master 
+--------------------------------
+
+Each Master process will initially try and contact all other nodes in
+the ensemble. Until more than half (in this case, two) nodes are alive
+and contactable, the configuration store will be unable to start, and
+the Flume Master will not be able to read or write configuration data.
+
+You can check the current state of the ensemble by inspecting the web page for any of the Flume Master machines which by default will be found at, for example, +http://masterA:35871+.
+
+==== Configuration Stores
+
+The Flume Master stores all its data in a *configuration store*. Flume has a pluggable configuration store architecture, and supports two implementations. 
+
+- The Memory-Backed Config Store (MBCS) stores configurations temporarily in memory. If the master node fails and reboots, all the configuration data will be lost. The MBCS is incompatible with distributed masters.  However, it is very easy to administer, computationally lightweight, and good for testing and experimentation.
+
+- The ZooKeeper-Backed Config Store (ZBCS) stores configurations   persistently and takes care of synchronizing them between multiple masters.
+
+.Flume and Apache ZooKeeper ************************************************ 
+Flume relies on the Apache ZooKeeper coordination platform to provide reliable, consistent, and persistent storage for node configuration data. A ZooKeeper ensemble is made up of two or more nodes which communicate regularly with each other to make sure each is up to date. Flume embeds a ZooKeeper server inside the Master process, so starting and maintaining the service is taken care of. However, if you have an existing ZooKeeper service running, Flume supports using that external cluster as well. ************************************************
+
+==== Which Configuration Store Should I Use?
+
+In almost all cases, you should use the ZBCS. It is more reliable and fault-tolerant, and will recover configurations after a restart. It is compatible with both standalone and distributed deployments of the Flume Master.
+
+The MBCS is appropriate if you are experimenting with Flume and can stand to lose configuration if the machine fails.
+
+ZBCS is the default configuration store. The choice of which configuration store to use is controlled by the +flume.master.store+ system property.
+
+[source,xml] 
+-------------------------------------- 
+<property>   
+<name>flume.master.store</name>   
+<value>zookeeper</value> 
+</property> 
+--------------------------------------
+
+If set to +memory+, the Flume Master will use MBCS instead. This is only supported in standalone mode. 
+
+==== Configuring the ZBCS
+
+Most deployments using the ZBCS can use Flume's default configuration. However, where more control over the precise configuration of the Flume Master is needed, there are several properties that you can set.
+
+Log Directory - +flume.master.zk.logdir+ :: To ensure reliability and the ability to restore its state in the event of a failure, ZBCS continually logs all updates it sees to the directory in +flume.master.zk.logdir+. This directory must be writable by the user as which Flume is running, and will be created if it doesn't exist at start-up time. WARNING: Do not delete this directory, or any files inside it. If deleted, all your configuration information will be lost.
+
+ZBCS Server Ports :: Each machine in the distributed Flume Master communicates with every other on the TCP ports set by +flume.master.zk.server.quorum.port+ and +flume.master.zk.server.election.port+. The defaults are 3182 and 3183 respectively. Note that these settings control both the port on which the ZBCS listens, and on which it looks for other machines in the ensemble. 
+
+ZBCS Client Port - +flume.master.zk.client.port+ :: The Flume Master process communicates with ZooKeeper (either on the same machine, or remotely on another Master server) via a client TCP port, which is set by +flume.master.zk.client.port+. The default is 3181.
+
+==== Gossip in Distributed Mode
+
+Flume Master servers also use a 'gossip' protocol to exchange information between themselves. Each server periodically wakes and picks another machine to send new data to. This protocol by default uses TCP port 57890, but this is controlled via the +flume.master.gossipport+ property:
+
+-------------------------------------- 
+<property>   
+<name>flume.master.gossip.port</name>   
+<value>57890</value> 
+</property> 
+--------------------------------------
+
+In standalone mode, there is no need to use gossip, so this port is unused. 
+
+
+==== Diagrams: How the Masters and Nodes talk to each other
+
+["graphviz", "master-zk-standalone.png"] 
+.Flume Master: Standalone Mode 
+--------------------------------------------------------------------- 
+digraph multi_collector {
+   label="Flume Master: Standalone Mode";
+       rankdir=LR;
+    edge [arrowhead="none"];
+   node [shape=none]   
+nodes ->masters [label="flume.master.config.port", fontsize=10];   
+masters -> zookeeper [label="flume.master.store.port", fontsize=10];
+  node [shape=Mrecord]
+   edge [style="bold", weight=10]
+   FM1 -> ZK1;
+
+
+   subgraph cluster_FM1 { label="Flume Master 1"; FM1; ZK1; }
+
+  subgraph cluster_ZK {     
+{ rank=same; ZK1 }        
+label="ZooKeeper Ensemble"; 
+
+        }
+
+  node [shape=none height=.15 fontsize=10]   
+edge [style="filled" arrowhead="none"];  
+ node1 -> FM1:w;   
+node2 -> FM1:w;   
+node3 -> FM1:w;   
+node4 -> FM1:w;   
+node5 -> FM1:w;   
+node6 -> FM1:w;   
+node7 -> FM1:w;   
+node8 -> FM1:w;   
+node9 -> FM1:w;
+
+ { rank=same; FM1;  masters }
+	     } 
+---------------------------------------------------------------------
+
+
+["graphviz", "master-zk-internal.png"] 
+.Flume Master: Distributed Mode 
+--------------------------------------------------------------------- 
+digraph multi_collector {   
+label="Flume Master: Distributed Mode";       
+rankdir=LR;    
+edge [arrowhead="none"];   
+node [shape=none]   
+nodes ->masters [label="flume.master.config.port", fontsize=10];   
+masters -> masters [label="flume.master.gossip.port", fontsize=10];   
+masters -> zookeeper [label="flume.master.zk.client.port", fontsize=10];   
+zookeeper -> zookeeper [label="flume.master.zk.server.{quorum,election}.port", fontsize=10];
+
+  node [shape=Mrecord]   
+edge [style="bold", weight=10]   
+FM1 -> ZK1;   
+FM2 -> ZK2;   
+FM3 -> ZK3;
+
+
+   subgraph cluster_FM1 { label="Flume Master 1"; FM1; ZK1; }  
+subgraph cluster_FM2 { label="Flume Master 2"; FM2; ZK2; }  
+subgraph cluster_FM3 { label="Flume Master 3"; FM3; ZK3; }
+
+  subgraph cluster_ZK {     
+{ rank=same; ZK1; ZK2; ZK3}        
+label="ZooKeeper Ensemble";      
+ edge  [style="filled" arrowhead="none"];     
+ZK1:e->ZK3:e;     
+ZK2:e->ZK3:e;     
+ZK1:e->ZK2:e;
+
+  }
+
+  subgraph cluster_Gossip {     
+{ rank=same; FM1; FM2; FM3}        
+label="Gossip";       
+edge  [style="filled" arrowhead="none"];     
+FM1:e->FM3:e;     
+FM2:e->FM3:e;     
+FM1:e->FM2:e;
+
+  }
+
+
+node [shape=none height=.15 fontsize=10]   
+edge [style="filled" arrowhead="none"];   
+node1 -> FM1:w;   
+node2 -> FM1:w;   
+node3 -> FM1:w;   
+node4 -> FM2:w;   
+node5 -> FM2:w;   
+node6 -> FM2:w;   
+node7 -> FM3:w;   
+node8 -> FM3:w;   
+node9 -> FM3:w;
+
+ { rank=same; FM1; FM2; FM3; masters }	      
+} 
+---------------------------------------------------------------------
+
+
+==== Configuring Flume Nodes to Connect to Multiple Master Servers
+
+One property needs to be set to configure a Flume Node to connect to multiple Masters: +flume.master.servers+.
+
+[source,xml] 
+-------------------------------------- 
+<property>   
+<name>flume.master.servers</name>   
+<value>masterA,masterB,masterC</value> 
+</property> 
+--------------------------------------
+
+The nodes connect over the port +flume.master.heartbeat.port+ on each machine in the Flume Master - this is the port that the Master servers listen on for node heartbeats.
+
+If a Master server fails, nodes will automatically fail over to the next randomly selected Master server that they can establish a connection to.
+
+=== External ZooKeeper Cluster
+
+In some cases you may want a ZBCS that relies on an externally managed ZooKeeper service. The most common example of this is where multiple services which rely on ZooKeeper are being used (Flume and Hbase for example). In the following example zkServer{A,B,C}:2181 should be replaced with the hostname/port of the ZooKeeper servers which make up your ensemble.
+
+.`conf/flume-site.xml` 
+[source,xml] 
+-------------------------------------- 
+<property>
+  <name>flume.master.zk.use.external</name>
+  <value>true</value>
+</property>
+
+<property>
+  <name>flume.master.zk.servers</name>
+  <value>zkServerA:2181,zkServerB:2181,zkServerC:2181</value>
+</property>
+-------------------------------------- 
+
+=== Section Summary 
+
+This section described installing, deploying, and configuring a set of
+Flume nodes in a fully distributed setting. You should now be able to
+collect streams of logs with Flume.
+
+You also used some roles for sources and sinks to connect nodes
+together.  You now have an understanding of the basics of setting up a
+set of Flume nodes.  Here's the new sources and sinks introduced in
+this subsection.
+
+// These tables are kinda gross, had to use asciidoc v8.2.7 style. 
+.Flume's Tiered Event Sources
+
++collectorSource[(_port_)]+ :: Collector source. Listens for data from
+agentSinks forwarding to port +_port_+.  If port is not specified, the
+node default collector TCP port, 35853.
+
+
+
diff --git a/flume-docs/src/docs/UserGuide/Glossary b/flume-docs/src/docs/UserGuide/Glossary
new file mode 100644
index 0000000..44ef96d
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Glossary
@@ -0,0 +1,44 @@
+﻿
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+[glossary]
+== Glossary
+
+Agent::
+A Flume node located at the start of a flow that captures data from 
+external sources, ready for feeding downstream. 
+
+Collector::
+A Flume node located at the end of a flow that delivers data to its eventual destination.
+
+Flow::
+A set of nodes wired together in sequence which together process data 
+from a single source into its eventual destination.
+
+Master::
+A service that controls the configuration of all nodes, and to 
+which all nodes report. 
+
+Sink::
+The place where a node sends its data after all processing is done.
+
+Source::
+The place where a node gets its data stream.
+
diff --git a/flume-docs/src/docs/UserGuide/Installation b/flume-docs/src/docs/UserGuide/Installation
new file mode 100644
index 0000000..1045dba
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Installation
@@ -0,0 +1,192 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+[[Installation]]
+== Installation
+
+There are currently three packaging options available for installing Flume: a tarball (TGZ), a RPM
+and a DEB file. The only prerequisite to running Flume is Java 1.6.
+
+.Installation Prerequisites
+****
+* A Unix-like system (tested on CentOS 5.3+ and Ubuntu 9.04+, Mac OS X)
+* Java 1.6.x (only tested with Sun JRE/JDK 1.6)
+****
+
+=== Installation by Tarball (TGZ)
+
+The tarball is a self-contained package containing everything needed
+ to do get Flume up and running on a Unix-like system.  Installing
+ from the tarball is as easy as unpacking it in an appropriate
+ directory. For instance on most Linux based systems this can be
+ achieved by typing:
+
+----
+$ (cd /usr/local/ && sudo tar -zxvf <path_to_flume.tgz> )
+----
+
+To complete the configuration of a tarball-based installation, you need to set the environment variable +$FLUME_CONF_DIR+ to be the +conf/+ subdirectory of the location that you installed Flume into - e.g.
+
+----
+$ export FLUME_CONF_DIR=/usr/local/flume/conf
+----
+
+//If a user is running older bin tools you can achieve the same objective by typing:
+//----
+//$ ( cd /opt && mkdir flume && cd flume && gzip -cd  <path_to_flume.tgz> | tar -xvf - )
+//----
+
+The tarball does not come with any scripts suitable for running Flume
+as a service or daemon. This makes the tarball distribution
+appropriate for ad-hoc installations and preliminary testing, but a
+more complete installation is provided by the binary packages.
+
+=== RPM or Deb Based install
+The RPMs and DEBs are much more convenient than a tarball based install in that they:
+
+* Handle dependencies
+* Provide for easy upgrades
+* Automatically install resources to conventional locations
+* Handle daemon startup and shutdown
+
+Both the RPMs and DEBs are broken up into three packages
+
+* +flume-core+ (Everything you need to run Flume)
+* +flume-master+ (Handles starting and stopping the Flume master as a service)
+* +flume-node+ (Handles starting and stopping the Flume node as a service)
+
+All Flume installations require the common code provided by +flume-core+. To install on a Debian-package compatible system:
+----
+$ sudo dpkg --install <path to flume core deb>
+----
+
+or with +apt-get+:
+----
+$ sudo apt-get install <path to flume core deb>
+----
+
+At this point, you should have everything necessary to run Flume, and the +flume+ command should be in your +$PATH+. You can test this by running:
+
+----
+$ flume
+----
+You should see something like
+----
+usage: flume command [args...]
+commands include: 
+  dump            Takes a specified source and dumps to console
+  node            Start a Flume node/agent (with watchdog)
+  master          Start a Flume Master server (with watchdog)
+  version         Dump flume build version information 
+  node_nowatch    Start a flume node/agent (no watchdog)
+  master_nowatch  Start a Flume Master server (no watchdog)
+  class <class>   Run specified fully qualified class using Flume environment (no watchdog)
+                   ex: flume com.cloudera.flume.agent.FlumeNode 
+  shell           Start the flume shell
+  killmaster      Kill a running master
+----
+
+.Can't find Flume?
+**************************
+If +flume+ is not found, and you installed from from a tarball, make sure that +$FLUME_HOME/bin+ is in your $PATH.
+**************************
+
+The final step in a package based install is to automatically start
+Flume via system runtime levels. The +flume-node+ and +flume-master+
+packages make this easy.  
+
+==== Starting Nodes Automatically
+
+On DEB based systems:
+----
+$ sudo dpkg --install <path to flume node deb>
+----
+or with +apt-get+:
+----
+$ sudo apt-get install <path to flume node deb>
+----
+
+On RPM based systems:
+----
+$ sudo rpm --install <path to flume node rpm>
+----
+
+==== Starting the Flume Master Automatically
+----
+$ sudo dpkg --install <path to flume master deb>
+----
+or with +apt-get+:
+----
+$ sudo apt-get install <path to flume master deb>
+----
+
+On RPM based systems:
+----
+$ sudo rpm --install <path to flume master rpm>
+----
+
+=== Running Flume
+Once flume is installed via a RPM or DEB you can start, stop or restart either the Flume Master or the Flume node via init scripts with the following commands:
+
+To control the Flume Master simply type:
+----
+$ /etc/init.d/flume-master <start | stop | restart >
+----
+
+To control the Flume node:
+----
+$ /etc/init.d/flume-node <start | stop | restart >
+----
+
+The node and master can also be run in the foreground directly with any of the installation methods with the +flume+ command. 
+
+To start the Flume node simply type:
+
+----
+$ flume node 
+----
+
+Or to start the Flume master simply type:
+----
+$ flume master
+----
+
+.Can I run master and node on the same machine?
+*****************
+When using Flume for the first time, it's common to try running both master and node on the same machine. This should work without a problem - both node and master can share the same configuration files and should not conflict on any ports they open. See the Pseudo-Distributed Mode section for more details.
+*****************
+
+=== Reference: What do the packages install and where?
+
+[grid="all"]
+`----------------------------`-----------------
+Resource                       Location
+Config Directory               /etc/flume/conf
+User Customizable Config File  /etc/flume/conf/flume-site.xml
+Daemon Log Directory           /var/log/flume
+Default Flume Home¶            /usr/lib/flume 
+Flume Master startup script¶   /etc/init.d/flume-master
+Flume Node startup script¶     /etc/init.d/flume-node
+Recommended TGZ Flume Home§    /usr/local/lib/flume
+----------------------------
+
+Key:
+§ Recommended but installation dependent
+¶ Provided by RPMS and DEBS
diff --git a/flume-docs/src/docs/UserGuide/Introduction b/flume-docs/src/docs/UserGuide/Introduction
new file mode 100644
index 0000000..34615ec
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Introduction
@@ -0,0 +1,380 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Abstract
+
+[abstract] 
+************************************************** 
+
+Flume is a distributed, reliable, and available service for
+efficiently collecting, aggregating, and moving large amounts of log
+data.  It has a simple and flexible architecture based on streaming
+data flows.  It is robust and fault tolerant with tunable reliability
+mechanisms and many failover and recovery mechanisms.  The system is
+centrally managed and allows for intelligent dynamic management. It
+uses a simple extensible data model that allows for online analytic
+applications.
+
+**************************************************
+
+[[Introduction]]
+== Introduction
+
+Flume is a distributed, reliable, and available service for
+efficiently moving large amounts of data soon after the data is
+produced.  This release provides a scalable conduit to move data
+around a cluster as well as reliable logging.
+
+The primary use case for Flume is as a logging system that gathers a
+set of log files on every machine in a cluster and aggregates them to
+a centralized persistent store such as the Hadoop Distributed File
+System (HDFS).  
+
+The system was designed with these four key goals in mind:
+
+* Reliability 
+* Scalability 
+* Manageability 
+* Extensibility
+
+This section provides a high-level overview of Flume's architecture
+and describes how the four design goals are achieved.
+
+=== Architecture
+
+Flume's architecture is simple, robust, and flexible.  The main
+abstraction in Flume is a stream-oriented *data flow*.  A data flow
+describes the way a single stream of data is transferred and processed
+from its point of generation to its eventual destination. Data flows
+are composed of *logical nodes* that can transform or aggregate the
+events they receive.  Logical nodes are wired together in chains to
+form a data flow. The way in which they are wired is called the
+logical node's *configuration*.
+
+Controlling all this is the *Flume Master*, which is a separate
+service with knowledge of all the physical and logical nodes in a
+Flume installation. The Master assigns configurations to logical
+nodes, and is responsible for communicating configuration updates by
+the user to all logical nodes.  In turn, the logical nodes
+periodically contact the master so they can share monitoring
+information and check for updates to their configuration.
+
+["graphviz", "architecture.png"]
+---------------------------------------------------------------------
+digraph multi_collector {
+  rankdir=LR;  
+
+
+  node [shape=none]
+  "agent tier" -> "collector tier" -> "storage tier"
+
+  node [shape=record];
+
+  { rank=same; "agent tier"; "agent A"; "agent B"; "agent C"; "agent D"; "agent E"; "agent F"; } 
+  { rank=same; "storage tier"; "master"; "HDFS" } 
+  { rank=same; "collector tier"; "collector" } 
+
+  collector [label="<A> collector A | <B> collector B | <C> collector C"];
+
+  master -> "agent A" [style="dashed" arrowhead="none"] ;
+  master -> "agent B" [style="dashed" arrowhead="none"] ;
+  master -> "agent C" [style="dashed" arrowhead="none"] ;
+  master -> "agent D" [style="dashed" arrowhead="none"] ;
+  master -> "agent E" [style="dashed" arrowhead="none"] ;
+  master -> "agent F" [style="dashed" arrowhead="none"] ;
+  master -> collector:A [style="dashed" arrowhead="none"] ;
+  master -> collector:B [style="dashed" arrowhead="none"] ;
+  master -> collector:C [style="dashed" arrowhead="none"] ;
+
+  collector:A -> HDFS [style="bold"]; 
+  collector:B -> HDFS [style="bold"]; 
+  collector:C -> HDFS [style="bold"]; 
+  "agent A" -> collector:A [style="bold"];
+  "agent B" -> collector:A [style="bold"];
+  "agent C" -> collector:B [style="bold"]; 
+  "agent D" -> collector:B [style="bold"];
+  "agent E" -> collector:C [style="bold"];
+  "agent F" -> collector:C [style="bold"];
+
+
+}
+---------------------------------------------------------------------
+
+// I think this should just use node here and introduce logical nodes
+// and physical nodes later
+
+The graph above shows a typical deployment of Flume that collects log
+data from a set of application servers.  The deployment consists of a
+number of _logical nodes_, arranged into three tiers. The first tier
+is the _agent_ tier.  Agent nodes are typically installed on the
+machines that generate the logs and are your data's initial point of
+contact with Flume.  They forward data to the next tier of _collector
+nodes_, which aggregate the separate data flows and forward them to
+the final _storage tier_.
+
+For example, the agents could be machines listening for +syslog+ data
+or monitoring the logs of a service such as a web server or the Hadoop
+JobTracker. The agents produce streams of data that are sent to the
+collectors; the collectors then aggregate the streams into larger
+streams which can be written efficiently to a storage tier such as
+HDFS.
+
+Logical nodes are a very flexible abstraction. Every logical node has
+just two components - a _source_ and a _sink_. The source tells a
+logical node where to collect data, and the sink tells it where to
+send the data. The only difference between two logical nodes is how
+the source and sink are configured. Both source and sink can
+additionally be configured with _decorators_ which perform some simple
+processing on data as it passes through. In the previous example, the
+collector and the agents are _running the same node software_.  The
+Master assigns a configuration to each logical node at run-time - all
+components of a node's configuration are instantiated dynamically at
+run-time, and therefore configurations can be changed many times
+throughout the lifetime of a Flume service without having to restart
+any Java processes or log into the machines themselves. In fact,
+logical nodes themselves can be created and deleted dynamically.
+
+
+The source, sink, and optional decorators are a powerful set of
+primitives. Flume uses this architecture to provide per-flow data
+properties (for example durability guarantees, compression, or
+batching), or to compute event metadata, or even generate new events
+that are inserted into data flow.  A logical node can also send data
+downstream to several logical nodes.  This allows for multiple flows
+and each subflow can potentially be configured differently.  For
+example, it’s possible to have one flow be a collection path,
+delivering data reliably to a persistent store, while another branch
+computes lightweight analytics to be delivered to an alerting system.
+
+// I think this box is too early.
+
+.Logical and Physical Nodes
+***********************
+It's important to make the distinction between _logical nodes_ and
+_physical nodes_. A physical node corresponds to a single Java process
+running on one machine in a single JVM instance. Usually there is just
+one physical node per machine. Physical nodes act as containers for
+_logical nodes_, which are wired together to form data flows. Each
+physical node can play host to many logical nodes, and takes care of
+arbitrating the assignment of machine resources between them. So,
+although the agents and the collectors in the preceding example are
+_logically_ separate processes, they could be running on the same
+_physical_ node. Flume gives users the flexibility to specify where
+the computation and data transfer are done. The remainder of this
+guide describes logical nodes unless stated otherwise.
+***********************
+
+=== Reliability
+
+Reliability, the ability to continue delivering events in the face of
+failures without losing data, is a vital feature of Flume. Large
+distributed systems can and do suffer partial failures in many ways -
+physical hardware can fail, resources such as network bandwidth or
+memory can become scarce, or software can crash or run slowly. Flume
+emphasizes fault-tolerance as a core design principle and keeps
+running and collecting data even when many components have failed.
+
+Flume can guarantee that all data received by an agent node will
+eventually make it to the collector at the end of its flow as long as
+the agent node keeps running.  That is, data can be *reliably*
+delivered to its eventual destination.
+
+However, reliable delivery can be very resource intensive and is often
+a stronger guarantee than some data sources require. Therefore, Flume
+allows the user to specify, on a per-flow basis, the level of
+reliability required. There are three supported reliability levels:
+
+* End-to-end
+* Store on failure 
+* Best effort
+
+.A Note About Reliability
+******************
+Although Flume is extremely tolerant to machine, network, and software
+failures, there is never any such thing as '100% reliability'. If all
+the machines in a Flume installation were irrevocably destroyed in
+some terrible data center incident, all copies of Flume's data would
+be lost and there would be no way to recover them. Therefore all of
+Flume's reliability levels make guarantees about data delivery 'until
+some maximum number of failures have occurred'. Flume's failure modes
+- in terms of what can fail and what will keep running if they do -
+are described in detail later in this guide.
+******************
+
+The *end-to-end* reliability level guarantees that once Flume accepts
+an event, that event will make it to the endpoint - as long as the
+agent that accepted the event remains live long enough. The first
+thing the agent does in this setting is write the event to disk in a
+''write-ahead log'' (WAL) so that, if the agent crashes and restarts,
+knowledge of the event is not lost. After the event has successfully
+made its way to the end of its flow, an acknowledgment is sent back to
+the originating agent so that it knows it no longer needs to store the
+event on disk. This reliability level can withstand any number of
+failures downstream of the initial agent.
+
+The *store on failure* reliability level causes nodes to only require
+an acknowledgement from the node one hop downstream.  If the sending
+node detects a failure, it will store data on its local disk until the
+downstream node is repaired, or an alternate downstream destination
+can be selected.  While this is effective, data can be lost if a
+compound or silent failure occurs.
+
+The *best-effort* reliability level sends data to the next hop with no
+attempts to confirm or retry delivery.  If nodes fail, any data that
+they were in the process of transmitting or receiving can be
+lost. This is the weakest reliability level, but also the most
+lightweight.
+
+.Eliminating Single Points of Failure
+************************
+Even when using the end-to-end reliability level, a Flume flow can
+fail to make progress if there aren't any nodes available to process
+events. The data will continue to be collected by the agents, but the
+agents may not be able to deliver the data downstream until a suitable
+node comes online. In order to maintain high availability, Flume
+allows flows to fail over from one downstream node to another without
+any user intervention. The Flume Master can also be replicated for
+high availability which means that both the data plane and the control
+plane can tolerate a number of faults.
+************************
+
+=== Scalability 
+
+Scalability is the ability to increase system performance linearly -
+or better - by adding more resources to the system.  Flume's goal is
+horizontal scalability -- the ability to incrementally add more
+machines to the system to increase throughput. A key performance
+measure in Flume is the number or size of events entering the system
+and being delivered.  When load increases, it is simple to add more
+resources to the system in the form of more machines to handle the
+increased load.
+
+As seen in the preceding example installation, there are three
+separate components of Flume that require different approaches to
+scalability: the collector tier, the master, and the storage tier.
+
+The collector tier needs to be able to scale in order to handle large
+volumes of data coming from large numbers of agent nodes.  This
+workload is write heavy, partitionable, and thus parallelizable.  By
+adding more machines to the collector tier, you can increase the
+number of agents and the maximum available throughput of the system.
+
+An individual collector can usually handle many agents (up to
+hundreds) because each individual agent often produces only small
+amounts of log data compared to the full bandwidth available to the
+collector. Therefore, Flume balances flows from agents across
+different collectors. (One flow from an agent will talk to the same
+collector.) Flume uses a randomized algorithm to evenly assign lists
+of collectors to flows.  This automatically spreads the load, and also
+keeps the load spread in the case where a collector fails.
+
+As the number of nodes in the system increases, the volume of traffic
+on the control path to and from the Flume Master may become a
+bottleneck. The Flume Master also supports horizontal scaling by
+adding more machines - although just a small number of commodity
+servers can serve a large installation of nodes. The state of the
+Flume Master is kept synchronized and fully replicated, which ensures
+that it is both fault tolerant and highly scalable.
+
+Finally, Flume can only write data through a flow at the rate that the
+final destinations can accept. Although Flume is able to buffer data
+inside a flow to smooth out high-volume bursts, the output rate needs
+to be equal on average to the input rate to avoid log jams. Thus,
+writing to a scalable storage tier is advisable.  For example, HDFS
+has been shown to scale to thousands of machines and can handle many
+petabytes of data.
+
+=== Manageability
+
+Manageability is the ability to control data flows, monitor nodes,
+modify settings, and control outputs of a large system.  Manually
+managing the data flow from the sources to the end point is tedious,
+error prone, and a major pain point.  With the potential to have
+thousands of log-generating applications and services, it's important
+to have a centralized management point to monitor and change data
+flows, and the ability to dynamically handle different conditions or
+problems.
+
+The Flume Master is the point where global state such as the data
+flows can be managed. Via the Flume Master, users can monitor flows
+and reconfigure them on the fly.  The Flume Master has the information
+required to automatically respond to system changes such as load
+imbalances, partial failures, or newly provisioned hardware.
+
+You can dynamically reconfigure nodes by using the Flume Master.
+Although this guide describes examples of traditional three-tier
+deployments, the flexibility of the nodes allow for arbitrary node
+topologies.  You can reconfigure nodes by using small scripts written
+in a flexible dataflow specification language, which can be submitted
+via the Flume Master interface.
+
+You can administer the Flume Master by using either of two interfaces:
+a web interface or the scriptable Flume command shell.  The web
+interface provides interactive updates of the system’s state.  The
+shell enables administration via manually crafted scripts or
+machine-generated scripts.
+
+=== Extensibility
+
+Extensibility is the ability to add new functionality to a system. For
+example, you can extend Flume by adding connectors to existing storage
+layers or data platforms.  This is made possible by simple interfaces,
+separation of functional concerns into simple composable pieces, a
+flow specification language, and a simple but flexible data model.
+
+Flume provides many common input and output connectors.  When new
+input connectors (sources) are added, extra metadata fields specific
+to that source can be attached to each event it produces.  Flume
+reuses the common components that provide particular reliability and
+resource usage properties.  Some general sources include files from
+the file system, syslog and syslog-ng emulation, or the standard
+output of a process.  More specific sources such as IRC channels and
+Twitter streams can also be added.  Similarly, there are many output
+destinations for events.  Although HDFS is the primary output
+destination, events can be sent to local files, or to monitoring and
+alerting applications such as Ganglia or communication channels such
+as IRC.
+
+To enable easy integration with HDFS, MapReduce, and Hive, Flume
+provides simple mechanisms for output file management and output
+format management.  Data gathered by Flume can be processed easily
+with Hadoop and Hive.
+
+=== Section summary 
+
+The preceding Introduction section describes the high level goals and
+features of Flume. The following sections of this guide describe how
+to set up and use Flume: 
+
+* Step-by-step tutorial that introduces a single Flume node
+* Introduction to the Flume Master and a pseudo-distributed mode that
+  includes multiple nodes coordinated by the Flume Master
+* Description of a fully-distributed setup that also removes single
+  points of failure 
+* Flume use cases and a description of how to integrate Flume with
+  existing sources of data 
+* How to set up Flume's output so that integration with heavyweight
+  analysis systems such as Hadoop and Hive 
+* How to deploy Flume, set up arbitrary flows, and a specification of
+  Flume's data flow specification language 
+* Catalog of components available via the language 
+* A description of experimental features 
+* Troubleshooting information
+
+
diff --git a/flume-docs/src/docs/UserGuide/LogicalNodes b/flume-docs/src/docs/UserGuide/LogicalNodes
new file mode 100644
index 0000000..a562ebf
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/LogicalNodes
@@ -0,0 +1,357 @@
+﻿
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== Logical Configurations
+
+Manually configuring nodes in Flume is manageable for a small number of nodes, 
+but can become burdensome for an operator as demands inevitably grow.  
+Ideally, the operator only has to assign a role to a particular machine.  
+Because configuration management is centralized via the Master, the Master 
+potentially has all the information necessary to intelligently create a node 
+topology and isolate flows of data from each other.
+
+To explain how this can be done, the concept of a *logical node* is 
+introduced. To manage communications between logical nodes, the concepts of 
+*logical sources* and *logical sinks* are introduced.  To isolate different 
+groups of nodes, the concept of a *flow* is introduced that allows you to 
+group agents and collectors into separate and isolated groups.
+
+==== Logical Nodes
+
+The logical node abstraction allows for each JVM instance (a physical node) to 
+contain multiple logical nodes.  This allows for the processing of many source 
+sink combinations on many threads of execution to occur on a single JVM 
+instance.
+
+Each logical node has a name that may be completely different from its 
+physical name or hostname.  You now need new operations that enable you to 
+spawn a new node, map logical nodes to physical nodes, and decommission 
+existing logical nodes.
+
+NOTE: The following commands are entered via the web interface using the "raw 
+command" web page on the Master.  You might prefer using the Flume command 
+shell (described in a later section) for these operations.  The same commands 
+described in this section can be entered in web interface or entered at the 
+command shell by prefixing the command with 'exec'.
+
+Suppose that initially you know you want an agent-collector topology, but you 
+don’t know the particular names of the exact machines.  For now, you can 
+specify the configuration of the logical nodes without specifying any physical 
+machine names.
+
+----
+agent1 : _source_ | autoBEChain ; 
+collector1 : autoCollectorSource | collectorSink("hdfs://....") ; 
+----
+
+Later you learn that host1 is the name of the agent1 machine and host2 is the 
+name of the collector machine.  You can 'map' logical nodes onto the
+physical Flume instances on host1 and host2 by issuing the following map
+commands:
+
+----
+map host1 agent1
+map host2 collector1
+----
+
+Afterwards, the node status table should display a new row of information for 
+each logical node.  Each logical node reports its own execution state, 
+configuration, and heartbeat.  There is also a new entry in the logical node 
+mapping table showing that the logical node has been placed on the specified 
+physical node. To configure the node's sources and sinks, use exactly the same 
+mechanisms described in the previous sections.
+
+You can also remove a logical node by using the 'decommission' command. 
+Suppose you no longer needed agent1 and wanted to "turn it off".  You can do 
+so by entering the following command:
+
+----
+decommission agent1
+----
+
+This terminates the thread and removes the configuration associated with a 
+logical node, and the mapping between the logical node and physical node.
+
+You can also move a logical node from one physical node to another by first 
+unmapping a logical node and then mapping it on another physical node.  In
+this scenario, you change the collector1 from being on host2 to host3.  
+
+----
+unmap host2 collector1
+----
+
+At this point, the logical node mapping is removed, and collector1 is not 
+active anywhere. You can then map collector1 onto host3 by using the map
+command:
+
+----
+map host3 collector1
+----
+
+NOTE: Logical nodes are not templates -- if you want to have the same
+source/sink pairs on a particular physical node, you need to have a
+logical node for each.  When adminstering many logical nodes it is
+often useful to write a script that generates configurations and
+unique individual logical node names.  Using the part of a host name
+is a common pattern.
+
+ 
+==== Logical Sources and Logical Sinks
+
+WARNING: The logical sources and logical sinks feature does not
+currently work when using multiple masters.
+
+In the previous example, we used two abstractions under-the-covers that allow 
+the specifications of a graph topology for communications 'without having to 
+use physical hostnames and ports'. These abstractions -- the *logical source* 
+and *logical sink* -- allow you to create a different graph topology without 
+having to know physical machines until they are mapped.
+
+Suppose you have two nodes producing data and sending it to the consumer:
+
+----
+dataProducer1 : console | logicalSink("dataConsumer") ;
+dataProducer2 : console | logicalSink("dataConsumer") ;
+dataConsumer : logicalSource | console ;
+----
+
+Note that in this example, the destination argument is the *logical name* of 
+the node and not a specific host/port combination.
+
+To implement these features, there is a generalized mechanism where users 
+enter logical configurations that are 'translated' by the Master to a physical 
+configuration.
+
+When the logical nodes get mapped to physical nodes:
+
+----
+map host1 dataProducer1
+map host2 dataProducer2
+map host3 dataConsumer
+----
+
+and after the Master learns the host names (the host1, host2, and host3 
+machine’s heartbeat against the Master), the Master has enough information to 
+translate configurations with physical hostnames and ports.  A possible 
+translation would replace the logicalSource with a rpcSources and the 
+logicalSink with an rpcSinks:
+
+----
+dataProducer1 : console | rpcSink("host3",56789) ;
+dataProducer2 : console | rpcSink("host3",56789) ;
+dataConsumer : rpcSource(56789) | console ;
+----
+
+In fact, auto agents and collectors, are another example of *translated 
+sources and sinks*.  These translate auto*Chain sinks and collectorSource into 
+a configuration that uses logicalSinks and logicalSources which in turn are 
+translated into physical rpcSource and rpcSinks instances.
+
+TIP: Translations are powerful and can be fairly smart; if new collectors are 
+added, they will become new failover options.  If collectors are removed, then 
+the removed collectors will be automatically replaced by other failover nodes.
+
+
+==== Flow Isolation
+
+WARNING: The automatic flow isolation feature does not currently work
+when using multiple masters.
+
+What happens if you want collect different kinds of data from the same 
+physical node?  For example, suppose you wanted to collect httpd logs as well 
+as syslog logs from the same physical machine.  Suppose also you want to write 
+all of the syslog data from the cluster in one directory tree, and all of the 
+httpd logs from the cluster in another.
+
+One approach is to tag all the data with source information and then push all 
+the data down the same pipe.  This could then be followed by some post-
+processing that demultiplexes (demuxes) the data into different buckets.  
+Another approach is to keep the two sets of data logically isolated from each 
+other the entire time and avoid post processing.
+
+Flume can do both approaches but enables the latter lower-latency
+approach, by introducing the concept of grouping nodes into
+*flows*. Since there are logical nodes that allow for multiple nodes
+on a single JVM, you can have a node for each different kinds of
+produced data.
+
+The following example shows how flows can be used. Start by having six logical 
+nodes in the system.
+
+----
+fooNode1 : fooSrc | autoBEChain ;
+barNode1 : barSrc | autoBEChain ;
+forNode2 : fooSrc | autoBEChain ;
+barNode2 : barSrc | autoBEChain ; 
+fooConsumer : autoCollectorSource | collectorSink("hdfs://nn/foodir") ;
+barConsumer : autoCollectorSource | collectorSink("hdfs://nn/bardir") ;
+----
+
+In this scenario, there are two physical machines that produce both
+kinds of data -- foo data and bar data. You want to send data to
+single collector that collects both foo data and bar data and writes
+it to different HDFS directories.  You could then map the nodes onto
+physical nodes:
+
+----
+map host1 fooNode1
+map host1 barNode1
+map host2 fooNode2
+map host2 barNode2
+map host3 fooConsumer
+map host3 barConsumer
+----
+
+
+["graphviz", "single-flow.png"]
+.Flume Flows: Single Flow
+---------------------------------------------------------------------
+digraph single_flow {
+  label="A single flume flow";
+  rankdir=LR;  
+
+  node [shape=record];
+  HDFS;
+
+  node [shape=Mrecord]
+  edge [style="bold", weight=10]
+  fooNode1 -> fooConsumer -> HDFS;
+  fooNode1 -> barConsumer -> HDFS;
+  fooNode2 -> fooConsumer;
+  fooNode2 -> barConsumer;
+  barNode1 -> fooConsumer;
+  barNode1 -> barConsumer;
+  barNode2 -> fooConsumer;
+  barNode2 -> barConsumer;
+
+  subgraph cluster_host1 { label="host1"; fooNode1; barNode1; }
+  subgraph cluster_host2 { label="host2"; fooNode2; barNode2; }
+  subgraph cluster_host3 { label="host3"; fooConsumer; barConsumer }
+
+  node [shape=none]
+  "agent tier" -> "collector tier" -> "storage tier"
+}
+---------------------------------------------------------------------
+
+
+This setup essentially instantiates the first approach.  It mixes foo and bar 
+data together since the translation of autoBEChain would see two 
+collectorSources that the Master considers to be equivalent.  Foo data will 
+likely be sent to the barConsumer and bar data will likely be sent to 
+fooConsumer.
+
+You really wanted to separate sources of information into logically isolated 
+streams of data.  Flume provides a grouping abstraction called a *flow*.  A 
+flow groups particular logical nodes together so that the different logical 
+data types remain isolated.
+
+More concretely, it allows for a different failover chain for each kind of 
+data in the Flume cluster.  The auto*Chain based agents would only send data 
+to collectors in the same flow group.  This isolates data so that it only 
+flows to nodes within the group.
+
+Currently, the compact form of the configuration language does not
+allow you to specify flows.  Instead you must add an extra argument to
+the config command to specify a flow.
+
+This example shows commands that would be entered in the Flume shell
+without flow group information.  In this case all of the nodes are in
+the same flow.
+
+----
+exec config fooNode1 fooSrc autoBEChain
+exec config barNode1 barSrc autoBEChain
+exec config fooNode2 fooSrc autoBEChain
+exec config barNode2 barSrc autoBEChain
+exec config fooConsumer autoCollectorSource 'collectorSink("hdfs://nn/foodir")'
+exec config barConsumer autoCollectorSource 'collectorSink("hdfs://nn/bardir")'
+----
+
+Now using the following commands you can specify flows by adding an
+extra parameter after the node name.  In this example we have two
+flows: flowfoo and flowbar.  flowfoo contains fooNode1, fooNode2 and
+fooConsumer.  flowbar contains barNode1, barNode2 and barConsumer.
+
+----
+exec config fooNode1 flowfoo fooSrc autoBEChain
+exec config barNode1 flowbar barSrc autoBEChain
+exec config fooNode2 flowfoo fooSrc autoBEChain
+exec config barNode2 flowbar barSrc autoBEChain
+exec config fooConsumer flowfoo autoCollectorSource 'collectorSink("hdfs://nn/foodir")'
+exec config barConsumer flowbar autoCollectorSource 'collectorSink("hdfs://nn/bardir")'
+----
+
+
+["graphviz", "multi-flow.png"]
+.Flume Flows: Multiple Flows
+---------------------------------------------------------------------
+digraph multi_flow {
+  label="Multiple isolated flume flows";
+  rankdir=LR;  
+
+  node [shape=record];
+  HDFS;
+
+
+  node [shape=Mrecord]
+  edge [style="bold", weight=10]
+  fooNode1 -> fooConsumer -> HDFS;
+  fooNode2 -> fooConsumer;
+  barNode1 -> barConsumer -> HDFS;
+  barNode2 -> barConsumer;
+
+  subgraph cluster_host1 { label="host1"; fooNode1; barNode1; }
+  subgraph cluster_host2 { label="host2"; fooNode2; barNode2; }
+  subgraph cluster_host3 { label="host3"; fooConsumer; barConsumer }
+
+  node [shape=none]
+  "agent tier" -> "collector tier" -> "storage tier"
+
+}
+---------------------------------------------------------------------
+
+
+
+By using these commands, the data from fooNode1 and fooNode2 will only
+be sent to fooConsumer, and barNode1 and barNode2's data will only be
+sent to barConsumer.  Data from one node is not mixed with other data
+from other nodes unless explicitly connected.
+
+TIP: In practice it is a good idea to use different node names and
+different flow ids for different kinds of data.  When node names are
+reused, the default behavior is to attempt to recover from failures
+assuming that leftover data from a crashed execution or previous
+source/sink configuration version are still producing the same kind of
+data.
+
+==== Section Summary
+
+This section introduced logical nodes, logical sources, logical sinks, and 
+flows and showed how these abstractions enable you to automatically deal with 
+manageability problems.
+
+* Only one input source per physical node.  
+* Multiple sets of isolated flows.  
+* Being machine specific, having to know all physical host names and ports.
+
+The translation mechanism can be quite powerful.  When coupled with
+metrics information, this could be used to perform automated dynamic
+configuration changes.  A possible example would be to automatically
+commission or decommission new collectors to match diurnal traffic and
+load patterns.
diff --git a/flume-docs/src/docs/UserGuide/Plugins b/flume-docs/src/docs/UserGuide/Plugins
new file mode 100644
index 0000000..87a5867
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Plugins
@@ -0,0 +1,607 @@
+﻿
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== Extending via Sink/Source/Decorator Plugins
+
+An experimental plugin mechanism is provided that allows you to add new custom 
+sources, sinks, and decorators to the system.  
+
+.Two steps are required to use this feature.
+. First, add the jar with the new plugin classes to flume's classpath.
+  If the plugin requires DLL's/so's make sure these are in the
+  LD_LIBRARY_PATH (unix .so) or PATH (windows .dll)
+. Second, in +flume-site.xml+, add the class names of the new sources,
+  sinks, and/or decorators to the +flume.plugin.classes+ property.
+  Multiple classes can be specified by comma separating the list. Java
+  reflection is used to find some special static methods that add new
+  components to the system and data flow language's library.
+
+An example component has been "pluginified" -- the "HelloWorld"
+source, sink, and decorator. This plugin does something very simple;
+the source generates the text "hello world!" every three seconds, the
+sink writes events to a "helloworld.txt" text file, and the decorator
+prepends "hello world!" to any event it encounters.
+
+. cd into the +plugins/helloworld+ directory and type +ant+, a +helloworld_plugin.jar+ file will be generated
+. Add the following to flume-site.xml (create it if it doesn't already
+  exist)
+  "helloworld.HelloWorldSink,helloworld.HelloWorldSource,helloworld.HelloWorldDecorator"
+  to the +flume.plugin.classes+ property in +flume-site.xml+.
++
+IMPORTANT: if you use the provided +flume-site.xml.template+ file to
+create your +flume-site.xml+ be sure to comment out or remove any
+example properties contained in the sample template.
++
+.Example flume-site.xml contents
+[source,xml]
+----
+<configuration>
+  <property>
+    <name>flume.plugin.classes</name>
+    <value>helloworld.HelloWorldSink,helloworld.HelloWorldSource,helloworld.HelloWorldDecorator</value>
+    <description>Comma separated list of plugins</description>
+  </property>
+</configuration>
+----
+. Start the Flume master and at least one logical node in separate terminals
+.. in each terminal cd into the top-level flume directory, should be just above +plugins+
+.. Add +helloworld_plugin.jar+ to the FLUME_CLASSPATH in *both* terminals
++
+----
+export FLUME_CLASSPATH=`pwd`/plugins/helloworld/helloworld_plugin.jar
+----
+.. in terminal 1 run +bin/flume master+
+.. in terminal 2 run +bin/flume node -n hello1+
+. At this point the master and hello1 nodes should be started and will have loaded the plugin
++
+.You should see log output similar to the following in both master and hello1:
+----
+10/07/29 17:35:28 INFO conf.SourceFactoryImpl: Found source builder helloWorldSource in helloworld.HelloWorldSource
+10/07/29 17:35:28 INFO conf.SinkFactoryImpl: Found sink builder helloWorldSink in helloworld.HelloWorldSink
+10/07/29 17:35:28 INFO conf.SinkFactoryImpl: Found sink decorator helloWorldDecorator in helloworld.HelloWorldDecorator
+----
++
+TIP: Another way to verify that your plugin is loaded is to check if it is displayed on this page http://localhost:35871/masterext.jsp
++
+. Configure hello1
++
+TIP: The easiest way to do this is open the configuration page of the master in a browser, typically this link http://localhost:35871/flumeconfig.jsp
++
+.. load the helloworld source/sink into our +hello1+ node (the bottom text box, then submit button if you are using the master's web interface"
++
+----
+hello1: helloWorldSource() | helloWorldSink();
+----
+.. you could also try the hello world decorator
++
+----
+hello1: helloWorldSource() | { helloWorldDecorator() => helloWorldSink() };
+----
++
+In either case +hello1+ will output a +helloworld.txt+ file into it's current working directory. Every 3 seconds a new "hello world!" line will be output to the file.
+
+==== Semantics of Flume Extensions
+
+Flume uses threading to support reconfiguration and multiple logical
+nodes.  Sources, sinks, and decorator are extensions that can use
+queues for buffering and can use clock sleeps for periodic rolls and
+retries.  Because we use these blocking operations and because the
+internal concurrency control mechanisms we use can cause deadlocks or
+hangs, there are several rules that need to be followed and enforced
+by test cases in order to be a well-behaved source, sink, or
+decorator.
+
+NOTE: This section is a draft and not exhaustive or completely formal.
+More restrictions may be added to this in the future.
+
+.Semantics  of sources
+
+Sources must implement 4 methods
+
+* void open() throws IOException
+* Event next() throws IOException
+* void close() throws IOException
+* ReportEvent getReport()
+
+Along with these signatures, each of these methods can also throw
+RuntimeExceptions.  These exceptions indicate a failure condition and
+by default will make a logical node shutdown in ERROR state.  These
+error messages are user visible and it is important that they have
+actionable descriptions about the failure conditions without having to
+dump stack.  Thus, NullPointerExceptions are not acceptable -- they
+are not descriptive or helpful without their stack traces.
+
+Some examples of valid runtime exceptions include invalid arguments
+that prevent a source from opening and invalid state infractions
+(attempting to next on a closed source).
+
+===== Simple source semantics.
+
+["graphviz", "sourceStatesSimple.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+START -> CLOSED ;
+CLOSED -> OPEN [label="open()"];
+CLOSED -> CLOSED [label="close()"];
+CLOSED -> CLOSED [label="next() throws Exn"];
+OPEN -> OPEN [label="next()"];
+OPEN -> OPEN [label="open() throws Exn"];
+OPEN -> CLOSED [label="close()"];
+} 
+---------------------------------------------------------------------
+
+Simple sources are assumed to be sources whose open and close
+operation happen quickly and do not block for long periods of time.
+The longest pauses tolerated here on the order of 10s (default time
+for a failing DNS lookup).
+
+The constructor for the sources should be callable without grabbing
+resources that can block or that require IO such as network connectors
+or file handles.  If there are errors due to configuration settings
+that can be caught in the constructor, an IllegalArgumentException
+should be thrown.
+
++open()+ is a call that grabs resources for the source so that the
++next()+ call can be made.  The +open+ call of a simple source should
+attempt to fail fast. It can throw a IOException or a RuntimeException
+such as IllegalStateException.  Open should only be called on a CLOSED
+sink -- if a sink is opened twice, the second call should throw an
+IOException or IllegalStateException.
+
++next()+ is a call that gets an event from an open source.  If a
+source is not open, this should throw an IllegalStateException.  This
+call can and will often block. If +next()+ is blocking, a call to
++close()+ should unblock the +next()+ by having it exit cleanly
+returning null.  Many resources such as TCP-sockets (and ideally rpc
+frameworks) default to throwing an exception on blocked network reads
+(like a +next()+ call) when +close()+'d.
+
++close()+ is a call that releases resources that the open call of a
+source grabs.  The +close()+ call itself blocks until all resources
+are released.  This allows a subsequent +open()+ in the same thread to
+not fail due to resource contention (ex, closing a server socket on
+port 12345 should not return until port 12345 is ready to be bound
+again).
+
++getReport()+ returns a ReportEvent.  These values should be available
+regardless if the node is open or closed and this call should not
+block the other source by other calls (due to potential lock inversion
+issues).  The values retrieved are ideally atomically grabbed but this
+is not required as long as no errors are caused by racy execution.
+
+If a source is opened or closed multiple times, it is up to the source to
+determine if values are reset or persisted between open/close cycles.
+
+===== Buffered source semantics
+
+["graphviz", "sourceStatesBuffered.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+START -> CLOSED ;
+CLOSED -> CLOSED [label="close()"];
+CLOSED -> CLOSED [label="next() throws Exn"];
+CLOSED -> OPEN [label="open()"];
+OPEN -> OPEN [label="next()"];
+OPEN -> OPEN [label="open() throws Exn"];
+OPEN -> CLOSING [label="close()"];
+CLOSING-> CLOSING [label="next()"];
+CLOSING-> CLOSING [label="close()"];
+CLOSING-> CLOSING [label="open() blocks"];
+CLOSING-> CLOSED [label="flushed"];
+CLOSING-> CLOSED [label="failed"];
+} 
+---------------------------------------------------------------------
+
+Some sources have in memory queues or buffers stored persistently on
+disk.  The guiding principle here is that on +close()+, buffered
+sources should prevent new data from entering and attempt to flush the
+buffered data.  Also any subordinate threads should be released before
+close returns.  If no progress is being made on +close()+, for a given
+period of time (30s is the default currently) the controlling thread
+will for an Thread.interrupt() call.  The source should be able to
+handle InterruptedExceptions and percolate interrupted status up the
+returning call stack.
+
+NOTE: In v0.9.1 and v0.9.2, interruptions when caught should be
+handled by re-flagging the Thread's interrupted flag (a call to
++Thread.currentThread().interrupt()+) and then throwing an
+IOException.  The API for extensions will likely change in the future
+to throw either an IOException or an InterruptedException.
+
++open()+ no change.
+
++close()+ This call is usually made from a separate thread than the
+open() or next() calls.  Since this call should block until resources
+are freed, it should attempt to flush its buffers before returning.
+For example, so if a network source has some buffered data, the
+network connection should be closed to prevent new data from entering,
+and then the buffered data should be flushed.  The source should be
+able to handle InterruptedExceptions and percolate interrupted status
+up the returning call stack and indicate an error state.
+
+A +next()+ call while in CLOSING state should continue pulling values
+out of the buffer until it is empty.  This is especially true if the
+next() call is happening in the main driver thread or a subordinate
+driver thread.  One mechanism to get this effect is to add a special
+DONE event to the queue/buffer that indicates a clean exit.
+
++getReport()+ ideally includes metric information like the size of the
+queues in the source, and the number of elements in the queue.
+
+
+.Semantics of sinks and decorators
+
+Sinks and decorators must implement 4 methods
+
+* void open() throws IOException
+* void append(Event e) throws IOException
+* void close() throws IOException
+* ReportEvent getReport()
+
+Along with these signatures, each of these methods can also throw
+RuntimeExceptions.  Run-time exceptions indicate a failure condition
+and by default will make a logical node shutdown in ERROR state.
+These error messages are user visible and it is important that they
+have helpful descriptions about the failure conditions without having
+to dump stack.  Thus, NullPointerExceptions are not acceptable -- they
+are not descriptive or helpful without their stack traces.
+
+===== Simple Sinks.
+
+["graphviz", "sinkStatesSimple.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+START -> CLOSED ;
+CLOSED -> OPEN [label="open()"];
+CLOSED -> CLOSED [label="close()"];
+CLOSED -> CLOSED [label="append() throws Exn"];
+OPEN -> OPEN [label="append()"];
+OPEN -> CLOSED [label="close()"];
+OPEN -> OPEN [label="open() throws Exn"];
+} 
+---------------------------------------------------------------------
+
+Simple sinks are assumed to have open, close, and append operations
+that happen quickly and do not block for long periods of time.  The longest
+pauses tolerated here on the order of 10s (default time for a failing
+DNS lookup).
+
+The constructor for the sinks and decorators should be callable
+without grabbing resources that can block or that require IO such as
+network connectors or file handles.  If there are errors due to
+configuration settings that can be caught in the constructor, an
+IllegalArgumentException should be thrown.
+
++open()+ is a call that grabs resources for the sink or decorator so
+that the +append(Event)+ call can be made.  If there are errors due to
+configuration settings not detectable without IO in the constructor,
++open()+ should attempt to fail fast and throw a IOException or
+RuntimeException such as IllegalStateException or
+IllegalArgumentException.  Open should only be called on a CLOSED sink
+-- if a sink is opened twice, the second call should throw an
+IOException or IllegalStateException.
+
++append()+ is a call that delivers a event.  If a sink is not open,
+this should throw an IllegalStateException.
+
+If a normal decorator fails to open or to append because of an
+internal failure or a subsink fails to open, the decorator should
+release its resources attempt to close the subsink and then throw an
+exception.  There are some sink/decos that specifically manipulate
+these semantics -- this needs to be done with care.
+
++close()+ is a call that releases resources that the open call of a
+source grabs.  If +open()+ or +next()+ is blocking, a call to this
+should unblock the call and have them exit.  +close()+ should be
+called called on an open sink, but we allow a closed sink to have
++close()+ called on it without throwing an exception (generally LOG
+warning however).
+
++getReport()+ returns a ReportEvent.  These values should be available
+regardless if the node is open or closed and this call should not
+cause get blocked by other calls (due to potential lock inversion
+issues).  The values retrieved are ideally atomically grabbed but this
+is not required as long as no errors are caused by racy execution.  If
+a sink is opened or closed multiple times, it is up to the sink to
+determine if values are reset or persisted between open/close cycles.
+
+
+===== Buffered sink and decorator semantics
+
+["graphviz", "sinkStatesBuffered.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+START -> CLOSED ;
+CLOSED -> CLOSED [label="close()"];
+CLOSED -> OPEN [label="open()"];
+CLOSED -> CLOSED [label="append() throws Exn"];
+OPEN -> OPEN [label="append()"];
+OPEN -> OPEN [label="open() throw Exn"];
+OPEN -> CLOSING [label="close()"];
+CLOSING -> CLOSING [label="close()"];
+CLOSING -> CLOSING [label="open() blocks"];
+CLOSING -> CLOSING [label="append() throws Exn"];
+CLOSING-> CLOSED [label="success"];
+CLOSING-> CLOSED [label="failure"];
+} 
+---------------------------------------------------------------------
+
+Some sinks have queues or buffers stored in memory or persistently on
+disk.  The guiding principle here is that buffered sinks should
+attempt to flush its buffers when prompted to +close()+.  This needs
+to be balanced with the requirement that sinks and decorated sinks
+should attempt to close in a relatively quick fashion.
+
++open()+ Since this sink isn't open this generally means there is no
+buffered data.  However, an +open()+ on a sink or decorator with
+persistent data should attempt to recover data and enqueue it in the
++open()+ call. Examples of these include a DFO or WAL log recovering
+dfo/wla logs, when a network subsink is down.
+
+An +append()+ call may buffer data before sending it (such as a
+batching decorator).  A +close()+ call, should attempt to append
+buffered data to its (sub)sink before executing the close. Also, any
+subordinate threads should be stopped before shutdown.
+
+If no progress is being made on +close()+, for a given period of time
+(30s is the default currently) it will be interrupted and should
+handle abruptly exiting because of this interruption.
+
+===== Retries, sleeps, and unclean exits.
+
+["graphviz", "sinkStatesOpening.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+START -> CLOSED ;
+CLOSED -> CLOSED [label="close()"];
+CLOSED ->  OPENING [label="open()"];
+OPENING -> CLOSED [label="close()"];
+OPENING -> OPEN [label="success"];
+OPENING -> CLOSED [label="interrupt"];
+OPENING -> OPENING [label="append() blocks"];
+OPEN -> OPEN [label="append()"];
+OPEN -> OPEN [label="open() throws Exn"];
+OPEN -> CLOSED [label="interrupt"];
+OPEN -> CLOSED [label="close()"];
+} 
+---------------------------------------------------------------------
+Retry on open semantics
+
+
+["graphviz", "sinkStatesAppending.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector {   
+rankdir=LR;   
+START -> CLOSED ;
+CLOSED -> CLOSED [label="close()"];
+CLOSED ->  OPEN [label="open()"];
+CLOSED -> CLOSED [label="append() throws Exn"];
+OPEN -> APPENDING [label="append()"];
+APPENDING -> OPEN [label="success"];
+APPENDING -> APPENDING [label="open()"];
+APPENDING -> APPENDING [label="close() blocks"];
+APPENDING -> APPENDING [label="append() blocks"];
+OPEN -> OPEN [label="open() throws Exn"];
+OPEN -> CLOSED [label="close()"];
+OPEN -> CLOSED [label="interrupt"];
+APPENDING -> CLOSED [label="interrupt"];
+} 
+---------------------------------------------------------------------
+Retry on Append semantics
+
+Some decorators introduce retries and sleeps. An author who uses these
+needs to ensure that these operations behave well on open, append, and
+close.  When combined with buffering sinks, flushing a buffer on close
+may not be possible! (ex: wal trying to send data to a dead network
+connection).  This means these sinks/decos need a way to exit abruptly
+and report an error.  There are two operations that make these cases
+ need to be handled: unbounded retries and unbounded/long
+sleeps/await/blocking.
+ 
+Some decorators have potentially unbounded retry semantics.  For
+example, InsistentOpen, InsistentAppend, and FailoverSinks have the
+potential to retry +open()+ and +append()+ calls an unbounded number
+of times.  These decorators can also be wrapped with in others -- this
+means we need to be able to percolate the hard exit and bypass the
+retry logic.
+
+To do this we require that any sink/deco that has retry logic must
+check for hard exit before retrying.  These sinks must to propagate
+the hard exit interruption to its upstream decorators (in case they
+have retry logic!).
+
+Some sinks have the potential to backoff or sleep for long or
+potentially unbounded amounts of time.  Code using sleeps or
+synchronization operations such as waiting on latches
+(+CountDownLatch.await()+) or thread sleeps (+Thread.sleep()+) must
+properly handle interruptions.  Since these yielding operations are
+usually only used in retry operations (which meant there was a
+failure), the sink/deco needs to propagate the interruption and fail
+with error.
+
+There are some ramifications of these semantics.  Care must be taken
+with locking open, close, and append operations.  If there are any
+sleeps or blocking +open()+ operations (ex: InsistentOpen,
+FailoverSinks), ideally a close call will cause it to shutdown, it and
+the open call should get unblocked.  +append()+
+
+The sink signaled to be closed but blocked on +append()+ or +open()+
+should exit in a reasonable amount of time -- ideally within a few
+heartbeats (5s is the default, so ideally <30s).  If the sink exits
+and its buffers are empty, it should do a normal successful return.
+If there were unflushed events, it should return error by throwing an
+exception.  If there are subordinate threads, these should be
+terminated before close returns.
+ 
+// NOTE: If you want more information about writing your own plugins, 
+// please contact flume-dev@cloudera.org.
+
+////////
+HIDDEN FOR NOW, THIS IS HALF BAKED
+
+=== Ganglia Reporting
+
+First we need two nodes -- in this example they are run on the same machine.  
+
+----
+Terminal 1: $ bin/flume node 
+
+Terminal 2: $ bin/flume node -n localhost
+----
+
+We assume the node is called _host_.
+
+Next let's map a new logical node called _report_ to localhost.
+
+----
+map localhost report
+----
+
+Finally lets set the configuration of the three nodes (replacing _host_ with 
+the actual host name).
+
+----
+host : asciisynth(0,100) | { delay(250) => agentSink } ;
+report : reportPoller | [ console, 
+dganglia("LogicalNodeManager._node_.LogicalNode.Agent.NaiveFileWAL.appendSucce
+ss", "msgs", "long") ];
+localhost : collectorSource | collectorSink("file:///tmp/test/%H","blah",5000);
+----
+
+This setup synthesize random ascii data events, and sends one event 250s 
+through the agentSink pipeline.  These are received by the +localhost+ node 
+which write data into hourly directories.  Meanwhile, the +report+ node on 
++_host_+ is a separate logical node that periodically polls the physical node 
+for reports and sends them to the console and a +dganglia+ sink.  The 
++dganglia+ sends the deltas (rate information) between particular values to 
+the ganglia gmond daemon.
+
+
+==== Experimental Sources and Sinks
+
+.Flume's Plugin Sources [horizontal] +amqpPoll(_..._)+ :: AMPQ (Advanced 
+Message Queuing Protocol) based on polling
+
++amqpSub(_..._)+ :: AMPQ (Advanced Message Queuing Protocol) based on 
+subscribing
+
++pcapfile("_pcap file_")+ :: One shot pcap file source (packet capture ala 
+wireshark).  One packet is one event
+
++pcap("_filterspec_")+ :: Live filtered pcap source (package capture ala 
+wireshark).  One packet is one event.
+
+.Flume's Plugin Sinks
+
++ampq()+ :: AMQP sink.  Write to specified AMQP broker (this is not present.)
+
+NOTE: Graphing currently uses Google charts API and requires the browser to 
+external network connectivity
+
+.Flume's Reporter Sinks (Experimental):
+[horizontal]
++ganglia("_attr_","_units_","_type_"[,"_gmondservers_"])+ :: This sink reads 
+absolute value of the attribute +_attr_+ as a type +_type_+ where +_type_+ may 
+be +int+, +long+, +double+ or +string+.  One may optionally specify a list of 
+gmond servers to send metric data to.
+
++dganglia("_attr_","_units_","_type_"[,"_gmondservers_"])+ :: This sink reads 
+the attribute +_attr_+ as a type +_type_+ where +_type_+ may be +int+, +long+, 
+or +double+ and then emits the delta of the previous value.  One may 
+optionally specify a list of gmond servers to send metric data to.
+
++counter("_name_")+ :: counts number of events received, and registers report 
+to report manager as +_name_+.
+
++counterHistory("name", .... )+ :: generates historical count versus time 
+graph
+
++multigrep(...)+ :: makes histogram that counts number of events that matches 
+substring grep
+
++multigrepspec(...)+ :: makes histogram that counts number of events that 
+match substring grep (uses config file instead of inline)|
+
++regexhisto(...)+ :: uses regex and builds histogram based on matching group's 
+value
+
++regexhisto(...)+ :: uses regex and builds histogram based on matching group's 
+value (uses config file instead of inline)
+
+.Flume's Experimental/Debugging Sink Decorators (unsupported)
+[horizontal] 
++flakeyAppend(_p_)+ :: each append has a probability +_p_+ (where 0.0 &le; _p_ 
+&le; 1.0) of failing and throwing an exception.
+
++intervalFlakeyAppend(_n_)+ :: every +_n_+ th element fails by throwing an 
+exception
+
++intervalDroppyAppend(_n_)+ :: every +_n_+ th element fails by throwing an 
+exception
+
++inmem+ :: buffers events in memory and then flushes all on close. Good for 
+benchmarking.
+
++benchinject+ :: injects an extra benchmark begin event on open and a 
+benchmark end event on close
+
++benchreport("_name_")+ :: starts a benchmark when it receives a benchmark 
+begin event and stops a benchmark when it receives a corresponding benchmark 
+end event.  Benchmark events are not forwarded, but all other events are.
+
++mult(_n_)+ :: takes each event and sends it _n_ copies of it before sending 
+the next event.
+
+// unused, easier to just remove.
+// +latch(...)+ :: (Unsupported) adds a latch that blocks until
+// triggered.  This is only really useful for debugging and test cases.
+
+===== Deprecated 
+
+.Flume's Deprecated Sources
+
+These sources are "too specific" or perform poorly and are thus deprecated 
+(but still available).  They will likely be replaced with a more generic 
+mechanisms (similar to the output format mechanism).
+
+.Flume's Experimental Sources
+
+[horizontal]
++tpriosource(_port_)+ :: Thrift RPC base source on TCP port, port,
+prioritized by priority (higher priority first) and then age (older 
+messages first)
+
++thriftSource(_port_)+ :: Same as RPC but specifically thrift RPC
+
+.Flume's Deprecated Sinks
+
+Many of these sinks are left over from earlier iterations.
+
+[horizontal]
+
+*None listed here for now*
+
+ END OF HIDDEN FOR NOW HALF BAKED 
+///////////////// 
diff --git a/flume-docs/src/docs/UserGuide/PseudoDist b/flume-docs/src/docs/UserGuide/PseudoDist
new file mode 100644
index 0000000..b9a2942
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/PseudoDist
@@ -0,0 +1,471 @@
+﻿
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Pseudo-distributed Mode
+
+Flume is intended to be run as a distributed system with processes spread out 
+across _many_ machines.  It can also be run as several processes on a _single_ 
+machine, which is called ``pseudo-distributed'' mode.  This mode is useful for 
+debugging Flume data flows and getting a better idea of how Flume components 
+interact.
+
+The previous section described a Flume node and introduced the concept of 
+Flume sources.  This section introduces some new concepts required for a 
+distributed setup: the Flume *master* server, the specification of sources and 
+*sinks*, and connecting multiple Flume *nodes*.
+
+=== Starting Pseudo-distributed Flume Daemons
+
+There are two kinds of processes in the system: the Flume *master* and the 
+Flume *node*.  The Flume Master is the central management point and controls 
+the data flows of the nodes.  It is the single logical entity that holds 
+global state data and controls the Flume node data flows and monitors Flume 
+nodes.  Flume nodes serve as the data path for streams of events.  They can be 
+the sources, conduits, and consumers of event data.  The nodes periodically 
+contact the Master to transmit a heartbeat and to get their data flow 
+configuration.
+
+In order to get a distributed Flume system working, you must start a single 
+Flume Master and some Flume nodes that interact with the Master. You'll start 
+with a Master and one Flume node and then expand.
+
+==== The Master
+
+The Master can be manually started by executing the following command:
+
+----
+$ flume master
+----
+
+After the Master is started, you can access it by pointing a web browser to 
+http://localhost:35871/.  This web page displays the status of all Flume nodes 
+that have contacted the Master, and shows each node's currently assigned 
+configuration.  When you start this up without Flume nodes running, the status 
+and configuration tables will be empty.
+
+image::master-empty.png[]
+
+The web page contains four tables -- the 'Node status' table, the 'Node 
+configuration' table, the 'Physical/Logical Node mapping' table, and a 
+'Command history' table.  The information in these tables represent the 
+current global state of the Flume system.
+
+The Master's 'Node status' table contains the names of all of the Flume Nodes 
+talking to the Master, their current configuration version (initially "none"), 
+their status (such as IDLE), and when it last reported to the Master.  The 
+name of each Flume node should be the same as running +hostname+ from Unix 
+prompt.
+
+The Master's 'Node configuration' table contains the logical name of a node, 
+the configuration version assigned to it, and a specification of its sources 
+and its sinks.  Initially, this table is empty, but after you change values 
+you can view this web page to see the updates. There are two sets of columns -
+- the user entered version/source/sink, and translated version/source/sink. A 
+later section of this guide describes translated configs.
+
+The Master's 'Physical/Logical Node mapping' table contains the mapping of 
+logical nodes to their physical nodes.
+
+The Master's 'Command history' table contains the state of commands. In 
+general, *commands* modify the Master's global state.  Commands are processed 
+serially on a Master and are assigned a unique command ID number. Each command 
+has a state (for example, SUCCEEDED, FAILED, or PENDING), a command line, and 
+a message which often contains information about its execution attempt.
+
+==== The Flume Node
+
+To start a Flume node, invoke the following command in another terminal.
+
+----
+$ flume node_nowatch
+----
+
+NOTE: Normally, you start a node using +flume node+ or run the node as a 
+daemon.  For these examples, you disable the watchdog feature by starting with 
+the +node_nowatch+ option.  This option enables you to interact with a node 
+via the console. The other options disable +stdin+.
+
+To check whether a Flume node is up, point your browser to the 'Flume
+Node status page' at http://localhost:35862/.  Each node displays its
+own data on a single table that includes diagnostics and metrics data
+about the node, its data flows, and the system metrics about the
+machine it is running on.  If you have multiple instances of the flume
+node program running on a machine, it will automatically increment the
+port number and attempt to bind to the next port (35863, 35864, etc)
+and log the eventually selected port.
+
+If the node is up, you should also refresh the Master's status page (http://
+localhost:35871) to make sure that the node has contacted the Master. You 
+brought up one node (assume the node is named +_host_+), so you should have 
+one node listed in the Master's node status table, and an entry in the logical 
+node mapping table that links the +_host_+ logical node to the +_host_+ 
+physical nodes.
+
+=== Configuring a Node via the Master
+
+Requiring nodes to contact the Master to get their configuration enables you 
+to dynamically change the configuration of nodes without having to log in to 
+the remote machine to restart the daemon.  You can quickly change the node's 
+previous data flow configuration to a new one. 
+
+The following describes how to "wire" nodes using the Master's web interface.
+
+On the Master's web page, click on the config link. You are presented with two 
+forms.  These are web interfaces for setting the node's data flows.  When 
+Flume nodes contact the Master, they will notice that the data flow version 
+has changed, instantiate, and activate the configuration.
+
+For this example, you will do the steps you did in the Quick Start section.  
+Enter the following values into the "Configure a node" form, and then click 
+Submit.
+
+[grid="all"]
+`----------`---------------- 
+Node name: +_host_+
+Source:    +console+ 
+Sink:      +console+ 
+----------------------------
+
+Refresh the Master page and notice that the version stamp changed to a current 
+time, and that the src and sink fields of the configs updated.  After the 
+status changes to "ACTIVE", it is ready to receive console traffic.
+
+On the master, a node can be in one of several states:
+
+* HELLO : A new node instance initially contacted the master.
+* IDLE : A node has completed its configuration or has no configuration.
+* CONFIGURING: A node has received a configuration and is activating the configuration.
+* ACTIVE: A node is actively pulling data from the source and pushing data into the sink.
+* LOST: A node has not contacted the master for an extended period of time (default is after 10x the expected heartbeat period -- 50s by default)
+* DECOMMISSIONED: A node has been purposely decommissioned from a master.
+* ERROR: A node has stopped in an error state.
+
+On the terminal where your Flume node is running, you should be able to type a 
+few lines and then get output back showing your new log message.
+
+[grid="all"]
+`----------`----------------
+Node name: +_host_+
+Source:    +text("/etc/services")+
+Sink:      +console+ 
+----------------------------
+
+NOTE: You may need to press Enter in the Flume node console. 
+
+Or use these new values to tail a file:
+
+[grid="all"]
+`----------`----------------
+Node name: +_host_+
+Source:    +tail("/etc/services")+ 
+Sink:      +console+ 
+----------------------------
+
+You can now change the configuration of different nodes in the system to 
+gather data from a variety of sources by going through the Master.
+
+=== Introducing Sinks
+
+Thus far, you have seen that Flume has a variety of sources that generate or 
+accept new events that are fed into the system.  You have limited the output 
+of these messages to the +console+ sink.  As you would expect, Flume also 
+provides a wide variety of event *sinks* -- destinations for all of the 
+events.
+
+There are many possible destinations for events -- to local disk, to HDFS, to 
+the console, or forwarding across the network. You use the sink abstractions 
+as an interface for forwarding events to any of these destinations.
+
+You can connect different sources to different sinks by specifying the new 
+configuration and submitting it to the Master.  For example, with the data 
+flow below, you can make a copy of +/etc/services+.
+
+[grid="all"]
+`----------`----------------
+Node name: +_host_+
+Source:    +text("/etc/services")+
+Sink:      +text("services.copy")+ 
+----------------------------
+
+WARNING: The +text+ sinks overwrite if a file previously exists.
+
+Notice that the file is copied as is. Sinks have optional arguments for output 
+format, which you can use to write data in other serialization formats.  For 
+example, instead of copying a file using the default "raw" formatter, you can 
+format the output using other formatters such as "avrojson" (the Avro 
+serialization json format), "avrodata" (the Avro serialization binary data 
+format), or a "debug" mode (this is the formatter used by the console sink).
+
+If you enter:
+
+[grid="all"]
+`----------`----------------
+Node name: +_host_+
+Source:    +text("/etc/services")+ 
+Sink:      +console("avrojson")+ 
+----------------------------
+
+You get the file with each record in JSON format displayed to the console. 
+
+If you enter:
+
+[grid="all"]
+`----------`----------------
+Node name: +_host_+
+Source:    +text("/etc/services")+
+Sink:      +text("services.json", "avrojson")+ 
+----------------------------
+
+The newly written local +services.json+ file is output in avro's json format.
+
+.Default Arguments
+********** 
+
+Many sinks, sources, and decorators have arguments that modify their behavior.  
+Some of these arguments are required; others are optional and use a default.
+
+The following notation describes these parameters:
+
++_name_ (_req1_, _req2_[, _opt1_[, _opt2_=_foo_]] )+
+
+where a component with name _name_, has two required arguments, _req1_ and 
+_req2_. There are two optional arguments, _opt1_ and _opt2_.  However, _opt1_ 
+*must* be specified if _opt2_ is specified.  _foo_ in this case is the default 
+argument for _opt2_+.
+
+For the +console+ examples above, console could be specified as:
+
++console[(_format_="debug")]+
+
+and text could be specified as:
+
++text("_file_"[, _format_="raw"])+
+
+**********
+
+There are several sinks you can use. The following list is a subset; see the 
+Appendix for more sinks. 
+
+.Flume Event Sinks
+[horizontal] 
++null+  :: Null sink. Events are dropped.
+
++console[("format")]+ :: Console sink.  Display to console's stdout. The 
+"format" argument is optional and defaults to the "debug" output format.
+
++text("_txtfile_"[,"format"])+ :: Textfile sink.  Write the events to text 
+file +_txtfile_+ using output format "format".  The default format is "raw" 
+event bodies with no metadata.
+
++dfs("_dfsfile_")+ :: DFS seqfile sink.  Write serialized Flume events
+to a dfs path such as +hdfs://namenode/file+ or +file:///file+ in
+Hadoop's seqfile format.  Note that because of the HDFS write
+semantics, no data for this sink write until the sink is closed.
+
++syslogTcp("_host_",_port_)+ :: Syslog TCP sink.  Forward to events to +host+ 
+on TCP port +port+ in syslog wire format (syslog-ng compatible), or to other 
+Flume nodes setup to listen for syslogTcp.
+
+WARNING: Using +dfs+ has some restrictions and requires some extra
+setup.  Files contents will not become available until after the sink
+has been closed.  See the Troubleshooting section for details.
+
+
+=== Aggregated Configurations
+
+Using the form for single node configuration of a small number of
+machines is manageable, but for larger numbers it is more efficient to
+maintain or auto-generate the configuration for all of the machines in
+a single file. Flume allows you to set the configurations of many
+machines in a single aggregated configuration submission.
+
+Instead of the method you used in the "Configuring a Node via the Master" 
+section, put the following configuration line into the "Configure Nodes" form 
+and then submit:
+
+----
+host : text("/etc/services") | console ;
+----
+
+Or:
+
+----
+host: text("/etc/services") | text("services.copy");
+----
+
+The general format is:
+
+----
+<node1> : <source> | <sink> ;
+<node2> : <source> | <sink> ;
+<node3> : <source> | <sink> ;
+...
+----
+
+The remainder of this guide uses this format to configure nodes.
+
+=== Tiering Flume Nodes: Agents and Collectors
+
+A simple network connection is abstractly just another sink.  It would be 
+great if sending events over the network was easy, efficient, and reliable.  
+In reality, collecting data from a distributed set of machines and relying on 
+networking connectivity greatly increases the likelihood and kinds of failures 
+that can occur.  The bottom line is that providing reliability guarantees 
+introduces complexity and many tradeoffs.
+
+Flume simplifies these problems by providing a predefined topology and tunable 
+reliability that only requires you to give each Flume node a role.  One simple 
+Flume node topology classifies Flume nodes into two roles -- a Flume *agent* 
+tier and the Flume *collector* tier. The agent tier Flume nodes are co-located 
+on machines with the service that is producing logs.  For example, you could 
+specify a Flume agent configured to have +syslogTcp+ as a source, and 
+configure the syslog generating server to send its logs to the specified local 
+port.  This Flume agent would have an +agentSink+ as its sink which is 
+configured to forward data to a node in the collector tier.
+
+Nodes in the collector tier listen for data from multiple agents, aggregates 
+logs, and then eventually write the data to HDFS.
+
+WARNING: In the next few sections, all +_host_+ arguments currently use the 
+physical IP or DNS name of the node as opposed to the Flume node name (set by 
++-n name+ option). The default Flume node name is the host name unless you 
+override it on the command line.  
+
+To demonstrate the new sinks in pseudo-distributed mode, you will instantiate 
+another Flume node (a physical node) on the local box.  To do this, you need 
+to start a Flume node with some extra options.  The command line below starts 
+a physical node named +collector+ (+-n collector+):
+
+----
+$ flume node_nowatch -n collector
+----
+
+On the Master's web page, you should eventually see two nodes: +_host_+ and 
++collector+.  The Flume Node status web pages should be available at http://
+localhost:35862 and http://localhost:35863.  Port bindings are dependent on 
+instantiation order -- the first physical node instantiated binds on 35862 and 
+the second binds to 35863.
+
+Next, configure +collector+ to take on the role of a collector, set up 
++_host_+ to send data from the console to the collector by using the 
+aggregated multiple configuration form.  The agent uses the +agentSink+, a 
+high reliability network sink.  The collector node's source is configured to 
+be a +collectorSource+, and its sink is configured to be the +console+.
+
+----
+host : console | agentSink("localhost",35853) ;
+collector : collectorSource(35853) | console ;
+----
+
+["graphviz", "tiers-console.png"]
+---------------------------------------------------------------------
+digraph single_collector { 
+
+  rankdir=LR;  
+
+  node [shape=none]
+  "agent tier" -> "collector tier"  [label="35872", fontsize=10];
+  "collector tier"-> "storage tier"   [ label="xxx", fontsize=10];
+
+  node [shape=record];
+
+  { rank=same; "agent tier"; host; }
+  { rank=same; "collector tier"; "collector" } 
+
+  host ->   collector;
+ }
+---------------------------------------------------------------------
+
+
+
+When you type lines in +_host_+'s console, events are forwarded to the 
+collector.  Currently, there is a bit of latency (15s or so) before the 
+forwarded message shows up on +collector+.  This is actually a configurable 
+setting whose default is set to a value that is amenable for high event 
+throughputs.  Later sections describe how to tune Flume.
+
+You have successfully made a event flow from an agent to the collector.
+
+TIP: You can check to see if +agentSink+ is working by looking in the
+write- ahead logging directory for the sink.  The default location is
++/tmp/flume/ <nodename>+.  Because the OS periodically cleans this
+directory, this configuration property (+flume.agent.logdir+) should
+be changed a production environment.
+
+A more interesting setup is to have the agent tailing a local file (using the 
++tail+ source) or listening for local syslog data (using the +syslogTcp+ or 
++syslogUdp+ sources and modifying the syslog daemon's configuration).  Instead 
+of writing to a console, the collector would write to a +collectorSink+, a 
+smarter sink that writes to disk or HDFS, periodically rotates files, and 
+manages acknowledgements.
+
+["graphviz", "tiers-hdfs.png"] 
+--------------------------------------------------------------------- 
+digraph single_collector { 
+
+  rankdir=LR;  
+
+  node [shape=none]
+  "agent tier" -> "collector tier"  [ label="35872", fontsize=10];
+  "collector tier"-> "storage tier"   [ label="xxx", fontsize=10];
+
+  node [shape=record];
+
+  { rank=same; "agent tier"; host; }
+  { rank=same; "storage tier"; "HDFS" } 
+  { rank=same; "collector tier"; "collector" } 
+
+  host ->   collector -> HDFS;
+}
+---------------------------------------------------------------------
+
+The following configuration is for an agent that listens for syslog messages 
+and forwards to a collector which writes files to local directory +/tmp/flume/
+collected/+.
+
+----
+host : syslogTcp(5140) | agentSink("localhost",35853) ;
+collector : collectorSource(35853) | collectorSink("file:///tmp/flume/collected", "syslog");
+----
+
+In the following slightly modified configuration, the collector writes to an 
+HDFS cluster (assuming the HDFS nameNode is called +namenode+):
+
+----
+host : syslogTcp(5140) | agentSink("localhost",35853) ;
+collector : collectorSource(35853) | collectorSink("hdfs://namenode/user/flume/
+","syslog");
+----
+
+NOTE: There are no guarantees that data written to an HDFS file is
+durable until the HDFS file is properly closed.  Because of this, the
+collector sink periodically closes a file and creates a new one in
+HDFS.  The default time between file rolls (close then open new) is
+30s.  If you are writing data at low throughput (<2MB/s) you may want
+to increase the default time by modifying the
++flume.collector.roll.millis+ and +flume.agent.logdir.retransmit+ time
+properties in your flume-site.xml file.
+
+=== Section Summary
+
+This section describes how to start a Master and a node, and configure a node 
+via the Master. The next section describes a more concise way of specifying 
+many configurations, explains agents and collectors, and how to build an 
+agent-collector pipeline on a single machine in a three-tier topology.
+
diff --git a/flume-docs/src/docs/UserGuide/QuickStart b/flume-docs/src/docs/UserGuide/QuickStart
new file mode 100644
index 0000000..0db390d
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/QuickStart
@@ -0,0 +1,292 @@
+﻿
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+[[Quickstart]] 
+== Flume Single Node Quick Start
+
+In this section, you will learn how to get a single Flume node running and 
+transmitting data. You will also learn about some data *sources*, and how to 
+configure Flume flows on a per-node basis.
+
+Each logical node consists of a event-producing *source* and an event-
+consuming *sink*.  Nodes pull data from their sources, and push data out 
+through their sink.
+
+NOTE: This section assumes that the Flume node and Flume Master are running in 
+the foreground and not as daemons. You can stop the daemons by using '/etc/
+init.d/flume-master stop' and '/etc/init.d/flume-node stop'.
+
+
+=== Sources and the `dump` command
+
+Start by getting a Flume node running that echoes data written to standard 
+input from the console back out to the console on stdout. You do this by using 
+the +dump+ command.
+
+----
+$ flume dump console 
+----
+
+TIP: The Flume program has the general form `flume <command> [args ...]`.  If 
+you installed from the tarball package, the command can be found in 
++$FLUME_HOME/bin/+. If you installed from either RPM or DEB, then +flume+ 
+should already be in your path.
+
+TIP: The example above uses the `dump` command and `console` is the argument.  
+The command’s syntax is `flume dump <source> [<outputformat>]`. It prints data
+from +<source>+ to the console.  Optionally, an output format can be specified,
+otherwise the default text format is used.
+
+NOTE: Some flume configurations by default write to local disk.
+Initially the default is '/tmp/flume'.  This is good for initial
+testing but for production environments the +flume.agent.logdir+
+property should be set to a more durable location.
+
+NOTE: If the node refuses to run and exits with this message,
++agent.FlumeNode: Aborting: Unexpected problem with
+environment.Failure to write in log directory: '/tmp/flume'.  Check
+permissions?+, then check the +/ tmp/flume+ directory to make sure you
+have write permissions to it (change the owner or have the user join
+the group).  This is, by default, where various logging information is
+kept.
+
+
+You have started a Flume node where `console` is the source of incoming data.  
+When you run it, you should see some logging messages displayed to the 
+console.  For now, you can ignore messages about Masters, back-off and failed 
+connections (these are explained in later sections).  When you type at the 
+console and press a new line, you should see a new log entry line appear 
+showing the data that you typed. If you entered `This is a test`, it should 
+look similar to this:
+
+----
+hostname [INFO Thu Nov 19 08:37:13 PST 2009] This is a test 
+----
+
+To exit the program, press ^C.
+
+NOTE: Some sources do not automatically exit and require a manual ^C to
+exit.
+
+// TODO there are actually too many irrelevant events right now. 
+// Need to turn of heartbeat when using one shot option.
+
+==== Reading from a text file, `text`
+
+You can also specify other sources of events.  For example, if you want a text 
+file where each line represents a new event, run the following command. 
+
+----
+$ flume dump 'text("/etc/services")' 
+----
+
+This command reads the file, and then outputs each line as a new event.
+
+NOTE: The default console output escapes special characters with Java-style 
+escape sequences. Characters such as '"' and '\' are prefaced with an extra 
+'\'.
+
+NOTE: You can try this command with other files such as `/var/log/messages`, 
+`/var/log/syslog`, or `/var/log/hadoop/hadoop.log` also.  However, Flume must 
+run with appropriate permissions to read the files. 
+
+==== Tailing a file name, `tail` and `multitail`
+
+If you want to tail a file instead of just reading it, specify another source 
+by using `tail` instead of `text`.
+
+----
+$ flume dump 'tail("testfile")' 
+----
+
+This command pipes data from the file into Flume and then out to the console.
+
+This message appears: "File 'testfile' does not currently exist, waiting for 
+file to appear".
+
+In another terminal, you can create and write data to the file:
+
+----
+$ echo Hello world! >> testfile 
+----
+
+New data should appear.
+
+When you delete the file:
+
+----
+$ rm testfile 
+----
+
+The `tail` sink detects this.  If you then recreate the file, the `tail` 
+source detects the new file and follows it:
+
+----
+$ echo Hello world again! >> testfile 
+----
+
+You should see your new message appear in the Flume node console.
+
+You can also use the `multitail` source to follow multiple files by file name:
+
+----
+$ flume dump 'multitail("test1", "test2")' 
+----
+
+And send it data coming from the two different files:
+
+----
+$ echo Hello world test1! >> test1 
+$ echo Hello world test2! >> test2 
+----
+
+The tail source by default assumes `\n` as a delimiter, and excludes
+the delimiter from events.  There are optional line delimiter
+arguments that allow you to specify arbitrary regular expressions as
+delimiters and to specify if the delimiter should be part of the
+`prev` ious event, `next` event, or `exclude` d.
+
+Here are some examples and scenarios to illustrate:
+
+The following example tails a file that requires two or more
+consecutive new lines to be considered a delimiter. The newlines are
+excluded from the events.
+
+----
+tail("file", delim="\n\n+", delimMode="exclude")
+----
+
+This example tails a file and uses `</a>` as a delimiter, and appends
+the delimiter to the previous event.  This could serve as a
+quick-and-dirty xml record splitter.
+
+----
+tail("file", delim="</a>", delimMode="prev")
+----
+
+Finally, this example tails a file and uses the regex "\n\d\d\d\d" as
+a delimiter and appends the delimiter to the next event.  This could
+be used to gather lines from a stack dump in a log file that starts
+with four digits (like a year from a date stamp).
+
+----
+tail("file", delim="\\n\\d\\d\\d\\d", delimMode="next")
+----
+
+==== Synthetic sources, `synth`
+
+Here's one more example where you use the `synth` sources to generate events:
+
+----
+$ flume dump 'asciisynth(20,30)'
+----
+
+You should get 20 events, each with 30 random ASCII bytes.
+
+==== Syslog as a source, `syslogUdp` and `syslogTcp`
+
+As with files, you can also accept data from well known wire formats such as 
+syslog. For example, you can start a traditional syslog-like UDP server 
+listening on port 5140 (the normal syslog UDP port is the privileged port 514) 
+by running this command:
+
+----
+$ flume dump 'syslogUdp(5140)' 
+----
+
+You can feed the source data by using netcat to send syslog formatted data as 
+shown in the example below:
+
+----
+$ echo "<37>hello via syslog"  | nc -u localhost 5140 
+----
+
+TIP: You may need to press ^C to exit this command.
+
+NOTE: The extra +<37>+ is a syslog wireformat encoding of a message category 
+and priority level.
+
+Similarly, you can set up a syslog-ng compatible source that listens on TCP 
+port 5140 (the normal syslog-ng TCP port is the privileged port 514):
+
+----
+$ flume dump 'syslogTcp(5140)' 
+----
+
+And send it data:
+
+----
+$ echo "<37>hello via syslog" | nc -t localhost 5140 
+----
+
+TIP: You may need to press ^C to exit this command.
+
+Syslog backwards-compatibility allows data normally created from syslog, 
+rsyslog, or syslog-ng to be sent to and processed by Flume.
+
+=== Anatomy of an Event
+
+This section describes a number of sources of data that Flume can interoperate 
+with. Before going any further, it will be helpful for you to understand what 
+Flume is actually sending and processing internally.
+
+Flume internally converts every external source of data into a stream of 
+*events*. Events are Flume's unit of data and are a simple and flexible 
+representation. An event is composed of a *body* and *metadata*. The event 
+body is a string of bytes representing the content of an event. For example, a 
+line in a log file is represented as an event whose body was the actual byte 
+representation of that line. The event metadata is a table of key / value 
+pairs that capture some detail about the event, such as the time it was 
+created or the name of the machine on which it originated. This table can be 
+appended as an event travels along a Flume flow, and the table can be read to 
+control the operation of individual components of that flow. For example, the 
+machine name attached to an event can be used to control the output path where 
+the event is written at the end of the flow.
+
+An event's body can be up to 32KB long - although this limit can be controlled 
+via a system property, it is recommended that it is not changed in order to 
+preserve performance.
+
+=== Section Summary
+
+In this section, you learned how to use Flume's +dump+ command to print data 
+from a variety of different input sources to the console. You also learned 
+about the *event*, the fundamental unit of data transfer in Flume.
+
+The following table summarizes the sources described in this section.
+
+.Flume Event 
+Sources +console+ :: Stdin console 
+
++text("filename")+ :: One shot text file source.  One line is one event
+
++tail("filename")+ :: Similar to Unix's +tail -F+. One line is one event. 
+Stays open for more data and follows filename if file rotated.
+
++multitail("file1"[, "file2"[, ...]])+ :: Similar to +tail+ source but follows 
+multiple files.
+
++asciisynth(msg_count,msg_size)+ :: A source that synthetically generates 
+msg_count random messages of size msg_size.  This converts all characters into 
+printable ASCII characters.
+
++syslogUdp(port)+ :: Syslog over UDP port, port.  This is syslog compatible.
+
++syslogTcp(port)+ :: Syslog over TCP port, port. This is syslog-ng compatible.
+
diff --git a/flume-docs/src/docs/UserGuide/RateLimit b/flume-docs/src/docs/UserGuide/RateLimit
new file mode 100644
index 0000000..52a3907
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/RateLimit
@@ -0,0 +1,75 @@
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+=== Limiting Data Transfer Rate between Source-Sink pairs
+
+Flume has the ability to limit the rate of transfer of data between source-sink pairs.  
+This is useful to divide the network bandwidth non-uniformly among different 
+source-sink pairs based on the kind of logs they are transferring.  
+For example, you can ship some types of logs at a faster rate than others, or you can transfer logs at different rates at different times of the day. 
+
+
+Another example of when it is beneficial to limit the rate of data transfer is when the network (or a collector) recovers after a failure.  
+In this case, the agents might have a lot of data backed up to ship; if there is no limit on the transfer rate, 
+the agents can exhaust all of the resources of the collector and possibly cause it to crash.  
+
+
+In Flume, you can use a special sink-decorator called the +choke+ decorator to limit the rate of transfer of data between source-sink pairs.
+Each +choke+ decorator (called a +choke+) has to be assigned to a +choke-id+.  
+Here is an example of using a +choke+ between a source and sink of a node; the +choke-id+ of this +choke+ is "Cid".
+----
+node: source | { choke("Cid") => sink };
+----
+
+The +choke-ids+ are specific to a physical node. Before using a +choke+ on +node+, 
+you must register the +choke-id+ on the physical node containing the +node+.  
+You can register a +choke-id+ on a physical node using the +setChokeLimit+ command.  
+When registering a +choke-id+, you must also assign a rate limit (in KB/sec) to it.  
+Here is an example of registering a +choke-id+ "Cid" on a physical node +host+ and assigning a limit 1000 KB/sec.
+----
+exec setChokeLimit host Cid 1000
+----
+
+NOTE: You can also use setChokeLimit at the run-time to change the limit assigned to a +choke-id+. 
+
+The limit on the +choke-id+ specifies an upper bound on the rate at which the +chokes+ using that +choke-id+ can collectively transfer data.  
+In the preceding example, there is only one source-sink pair on the physical node +host+ that uses a +choke+ with +choke-id+ "Cid". Consequently, the rate of data transfer between that source-sink pair is limited to 1000 KB/sec. 
+
+NOTE: For the purpose of rate limiting, only the size of the event body is taken into account. 
+
+The +choke+ decorator works as follows: when +append()+ is called on the sink to which the +choke+ 
+is attached, the +append()+ call works normally if the amount of data transferred 
+(during a small duration of time) is within the limit assigned to the +choke-id+ corresponding to the +choke+.
+If the limit has been exceeded, then +append()+ is blocked for a small duration of time.
+
+Suppose there are multiple source-sink pairs using +chokes+ between them, and they are using 
+the same +choke-id+.  Suppose further that both +node1+ and +node2+ are logical nodes on the same physical node +host+, 
+and a +choke-id+ "Cid" with limit 1000KB/sec is registered on +host+.
+
+----
+node1: source1 | { choke("Cid") => sink1 };
+node2: source2 | { choke("Cid") => sink2 };
+----
+In this example, because both +source1-sink1+ and +source2-sink2+ pairs are using +chokes+ with the same +choke-id+ "Cid", the total data going across these source-sink pairs collectively is limited to 1000KB/sec. 
+Flume does not control how this limit will be divided between the source-sink pairs, 
+but it does guarantee that neither source-sink pair will starve.
+
+NOTE: If multiple source-sink pairs on the same physical node use chokes that have the same choke-id, 
+then there is no guarantee how the rate limit will be divided between these source-sink pairs.
+
+
diff --git a/flume-docs/src/docs/UserGuide/Security b/flume-docs/src/docs/UserGuide/Security
new file mode 100644
index 0000000..c613baa
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Security
@@ -0,0 +1,260 @@
+ 
+== Flume and HDFS Security Integration
+
+NOTE: This section is only required if you are using a Kerberized HDFS
+cluster.  If you are running CDH3b2 or a Hadoop version 0.21.x or
+earlier, you can safely skip this section.
+
+Flume's datapath needs to be able to interact with "secured" Hadoop
+and HDFS.  The Hadoop and HDFS designers have chosen to use the
+Kerberos V5 system and protocols to authenticate communications
+between clients and services.  Hadoop clients include users, MR jobs
+on behalf of users, and services include HDFS, MapReduce.
+
+In this section we will describe how setup up a Flume node to be a
+client as user 'flume' to a kerberized HDFS service.  This section
+will *not* talk about securing the communications between Flume nodes
+and Flume masters, or the communications between Flume nodes in a
+Flume flow.  The current implementation does not support writing
+individual isolated flows as different users.
+
+NOTE: This has only been tested with the security enhanced betas of
+CDH (CDH3b3+), and the MIT Kerberos 5 implementation.
+
+=== Basics
+
+Flume will act as a particular Kerberos principal (user) and needs
+credentials.  The Kerberos credentials are needed in order to interact
+with the kerberized service.
+
+There are two ways you can get credentials. The first is used by
+interactive users because it requires an interactive logon.  The
+second is generally used by services (like a Flume daemon) and uses a
+specially protected key table file called a 'keytab'.
+
+Interactively using the +kinit+ program to contact the Kerberos KDC
+(key distribution center) is one way is to prove your identity. This
+approach requires a user to enter a password.  To do this you need a
+two part principal setup in the KDC, which is generally of the form
++user@REALM.COM+.  Logging in via +kinit+ will grant a ticket granting
+ticket (TGT) which can be used to authenticate with other services.
+
+NOTE: this user needs to have an account on the namenode machine as
+well -- Hadoop uses this user and group information from that machine
+when authorizing access.
+
+Authenticating a user or a service can alternately be done using a
+specially protected 'keytab' file.  This file contains a ticket
+generating ticket (TGT) which is used to mutually authenticate the
+client and the service via the Kerberos KDC.
+
+NOTE: The keytab approach is similar to an "password-less" ssh
+connections.  In this case instead of an id_rsa private key file, the
+service has a keytab entry with its private key.
+
+Because a Flume node daemon is usually started unattended (via service
+script), it needs to login using the keytab approach.  When using a
+keytab, the Hadoop services requires a three part principal.  This has
+the form +user/host.com@REALM.COM+.  We recommend using +flume+ as the
+user and the hostname of the machine as the service.  Assuming that
+Kerberos and kerberized Hadoop has been properly setup, you just need
+to a few parameters to the Flume node's property file
+(flume-site.xml).
+
+----
+<property>
+<name>flume.kerberos.user</name>
+<value>flume/host1.com@REALM.COM </value>
+<description></description>
+</property>
+
+<property>
+<name>flume.kerberos.keytab</name>
+<value>/etc/flume/conf/keytab.krb5 </value>
+<description></description>
+</property>
+----
+
+In this case, +flume+ is the user, +host1.com+ is the service, and
++REALM.COM+ is the Kerberos realm.  The +/etc/keytab.krb5+ file contains
+the keys necessary for +flume/host1.com@REALM.COM+ to authenticate
+with other services.
+
+Flume and Hadoop provides a simple keyword (_HOST) that gets expanded
+to be the host name of the machine the service is running on.  This
+allows you to have one flume-site.xml file with the same
+flume.kerberos.user property on all of your machines.
+
+----
+<property>
+<name>flume.kerberos.user</name>
+<value>flume/_HOST@REALM.COM </value>
+<description></description>
+</property>
+----
+
+You can test to see if your Flume node is properly setup by running
+the following command.
+
+----
+flume node_nowatch -1 -n dump -c 'dump: console |  collectorSink("hdfs://kerb-nn/user/flume/%Y%m%D-%H/","testkerb");'
+----
+
+This should write data entered at the console to a kerberized HDFS
+with a namenode named kerb-nn, into a +/user/flume/YYmmDD-HH/+
+directory.
+
+If this fails, you many need to check to see if Flume's Hadoop
+settings (in core-site.xml and hdfs-site.xml) are using Hadoop's
+settings correctly.
+
+=== Setting up Flume users on Kerberos
+
+NOTE: These instructions are for MIT Kerb5.
+
+There are several requirements to have a "properly setup" Kerberos +
+HDFS + Flume.
+
+* Need to have a prinicipal for the Flume user on each machine.
+* Need to have a keytab that has keys for each principal on each machine.
+
+Much of this setup can be done by using the +kadmin+ program, and
+verified using the +kinit+, +kdestroy+, and +klist+ programs.
+
+==== Administering Kerberos principals
+
+First you need to have permissions to use the +kadmin+ program and the
+ability to add to principals to the KDCs.
+
+----
+$ kadmin -p <adminuser> -w <password>
+----
+
+If you entered this correctly, it will drop you do the kadmin prompt
+----
+kadmin:
+----
+
+Here you can add a Flume principal to the KDC
+
+----
+kadmin: addprinc flume
+WARNING: no policy specified for flume@REALM.COM; defaulting to no policy
+Enter password for principal "flume@REALM.COM": 
+Re-enter password for principal "flume@REALM.COM": 
+Principal "flume@REALM.COM" created.
+kadmin: 
+----
+
+You also need to add principals with hosts for each Flume node that
+will directly write to HDFS.  Since you will be exporting the key to a
+keytab file, you can use the -randkey option to generate a random key.
+
+----
+kadmin: addprinc -randkey flume/host.com
+WARNING: no policy specified for flume/host.com@REALM.COM; defaulting to no policy
+Principal "flume/host.com@REALM.COM" created.
+kadmin: 
+----
+
+NOTE: Hadoop's Kerberos implementation requires a three part principal
+name -- user/host@REALM.COM.  As a user you usually only need the user
+name, user@REALM.COM.  
+
+You can verify that the user has been added by using the +kinit+
+program, and entering the password you selected.  Next you can verify
+that you have your Ticket Granting Ticket (TGT) loaded.
+
+----
+$ kinit flume/host.com
+Password for flume/host.com@REALM.COM: 
+$ klist
+Ticket cache: FILE:/tmp/krb5cc_1016
+Default principal: flume/host.com@REALM
+
+Valid starting     Expires            Service principal
+09/02/10 18:59:38  09/03/10 18:59:38  krbtgt/REALM.COM@REALM.COM
+
+
+Kerberos 4 ticket cache: /tmp/tkt1016
+klist: You have no tickets cached
+$
+----
+
+You can ignore the Kerberos 4 info.  To "logout" you can use the
++kdestroy+ command, and then verify that credentials are gone by
+running +klist+.
+
+----
+$ kdestroy
+$ klist
+klist: No credentials cache found (ticket cache FILE:/tmp/krb5cc_1016)
+
+
+Kerberos 4 ticket cache: /tmp/tkt1016
+klist: You have no tickets cached
+$
+----
+
+Next to enable automatic logins, we can create a keytab file so that
+does not require manually entering a password.
+
+WARNING: This keytab file contains secret credentials that should be
+protected so that only the proper user can read the file.  After
+created, it should be in 0400 mode (-r--------) and owned by the user
+running the Flume process.
+
+Then you can generate a keytab file (int this example called
++flume.keytab+) and add a user +flume/host.com+ to it.
+
+----
+kadmin: ktadd -k flume.keytab flume/host.com
+----
+
+NOTE: This will invalidate the ability for flume/host.com to manually
+login of the account.  You could however have a Flume user does not
+use a keytab and that could log in.
+
+WARNING: +ktadd+ can add keytab entries for mulitple principals into a
+single file and allow for a single keytab file with many keys.  This
+however weakens the security stance and may make revoking credentials
+from misbehaving machines difficult.  Please consult with your
+security administrator when assessing this risk.
+
+You can verify the names and the version (KVNO) of the keys by running
+the following command.
+
+----
+$ klist -Kk flume.keytab
+Keytab name: FILE:flume.keytab
+KVNO Principal
+---- --------------------------------------------------------------------------
+   5 flume/host.com@REALM.COM (0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)
+   5 flume/host.com@REALM.COM (0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)
+   5 flume/host.com@REALM.COM (0xcccccccccccccccc)
+   5 flume/host.com@REALM.COM (0xdddddddddddddddd)
+----
+
+You should see a few entries and your corresponding keys in hex after
+your principal names.
+
+Finally, you can use +kinit+ with the +flume@REALM.COM+ principal to
+interactively do a Kerberos login and use the Hadoop commands to browse HDFS.
+
+----
+$ kinit flume
+Password for flume@REALM.COM:  <-- enter password
+$ hadoop dfs -ls /user/flume/
+----
+
+
+////
+
+Windows instructions.
+
+----
+ktpass // windows generate keytab file
+----
+
+
+////
diff --git a/flume-docs/src/docs/UserGuide/Troubleshooting b/flume-docs/src/docs/UserGuide/Troubleshooting
new file mode 100644
index 0000000..e0d7597
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/Troubleshooting
@@ -0,0 +1,130 @@
+﻿
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+=== Troubleshooting
+
+include::DefaultPorts[]
+
+==== What versions of Hadoop HDFS can I use?  How do I change this?
+
+Currently, there are constraints writing to HDFS.  A Flume node 
+can only write to one version of Hadoop.  Although Hadoop's HDFS API has been 
+fairly stable, HDFS clients are only guaranteed to be wire compatible with the 
+same major version of HDFS.  Cloudera's testing used Hadoop HDFS 0.20.x and HDFS 
+0.18.x.  They are API compatible so all that is necessary to switch versions 
+is to swap out the Hadoop jar and restart the node that will write to the 
+other Hadoop version.
+
+You still need to configure this instance of Hadoop so that it talks to the 
+correct HDFS nameNode.  You configure the Hadoop client settings (such as 
+pointers to the name node) the same way as with Hadoop dataNodes or worker 
+nodes -- modify and use `conf/core-site.xml`.
+
+By default, Flume checks for a Hadoop jar in +/usr/lib/hadoop+.  If it is not 
+present, it defaults to a jar found in its own lib directory, +/usr/lib/flume/
+lib+.
+
+
+==== Why doesn't a Flume node appear on Flume Master?
+
+If a node does not show up on the Master, you should first check to see if the 
+node is running.  
+
+----
+# jps | grep FlumeNode
+----
+
+You can also check logs found in +/usr/lib/flume/logs/flume-flume-node*.log+.  
+
+Common errors are error messages or warnings about being unable to contact 
+the masters.  This could be due to host misconfiguration, port 
+misconfiguration (35872 is the default heartbeat port), or fire walling 
+problems.
+
+Another possible error is to have a permissions problems with the local 
+machine's write-ahead log directory.  On an out-of-the-box setup, this is in 
+the +/tmp/flume/agent+ directory).  If a Flume Node is ever run as a user 
+other than +flume+, (especially if it was run as +root+), the directory needs 
+to be either deleted or its contents must have its permissions modified to 
+allow Flume to use it.
+
+==== Why is the state of a Flume node changing rapidly?
+
+Nodes by default start and choose their +hostname+ as their physical node 
+name.  Physical nodes names are supposed to be unique.  Unexpected results may 
+occur if multiple physical nodes are assigned the same name.
+
+==== Where are the logs for troubleshooting Flume itself?
+
+On Ubuntu-based installations, logs are written to +/usr/lib/logs/+.  
+
+Master logs   :: +/usr/lib/logs/flume-flume-master-_host_.log+ 
+Master stdout :: +/usr/lib/logs/flume-flume-master-_host_.out.*+ 
+Node logs :: +/usr/lib/ logs/flume-flume-node-_host_.log+ 
+Node stdout :: +/usr/lib/logs/flume-flume- node-_host_.out.*+
+
+
+==== What can I do if I get node failure due to out of file handles?
+
+There are two limits in Linux -- the max number of allowable open files 
+(328838), and max number of allowable open files for a user (default 1024).  
+Sockets are file handles so this limits the number of open TCP connections 
+available.
+
+On Ubuntu, need to add a line to +/etc/security/limits.conf+
+
+----
+<user> hard nofile 10000
+----
+
+The user should also add the following line to a `~/.bash_profile` to raise 
+the limit to the hard limit.
+
+----
+ulimit -n 10000
+----
+
+==== Failures due when using Disk Failover or Write Ahead Log
+
+Flume currently relies on the file system for logging mechanisms. You must 
+make sure that the user running Flume has permissions to write to the 
+specified logging directory.
+
+Currently the default is to write to /tmp/flume.  In a production system you 
+should write to a directory that does not automatically get deleted on reboot.  
+
+==== Can I write data to Amazon S3?
+
+Yes.  In the collector sink or dfs sinks you can use the +s3n://+ or +s3://+ 
+prefixes to write to S3 buckets.
+
+First you must add some jars to your CLASSPATH in order to support s3 writing. 
+Here is a set that Cloudera has tested (other versions will likely work as well):
+
+* commons-codec-1.3.jar
+* commons-httpclient-3.0.1.jar
+* jets3t-0.6.1.jar
+
++s3n+ uses the native s3 file system and has some limitations on individual 
+file sizes.  Files written with this method are compatible with other programs 
+like +s3cmd+.
+
++s3+ writes using an overlay file system must go through Hadoop for file 
+system interaction.
diff --git a/flume-docs/src/docs/UserGuide/UsingOutput b/flume-docs/src/docs/UserGuide/UsingOutput
new file mode 100644
index 0000000..d2231a2
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/UsingOutput
@@ -0,0 +1,310 @@
+﻿
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+
+== Using Data Collected by Flume
+
+The first goal of Flume is to collect data and reliably write it to HDFS.  
+Once data arrives, one wants the ability to control where and in what format 
+data is stored.  Flume provides basic output control mechanisms via the 
+properties configuration and in the dataflow language.  This gives the user 
+the ability to control the output format and output bucketing of incoming 
+data, and simplifies integration with other HDFS data consumers such as Hive 
+and HBase.
+
+Here are some example use cases:
+
+* When monitoring a web server, you want to bucket logs based on time,
+  the page hit, and the browser being used.
+* When tracking particular data nodes, you want to bucket logs based on
+  time and the data node name.
+* When tracking a feed of JIRA tickets from the Apache feed, you want
+  to group based on the project identifier or a particular person. 
+* When collecting data from scribe sources, you want to use its bucket
+  data based on its the event's category information.
+
+To support these kinds of features, Flume uses a simple data model,
+provides a mechanism for bucketing events, and also provides basic
+extraction operations for specifying custom bucketing discriminators.
+
+=== The Data Model of a Flume Event
+
+A Flume event has these six main fields: 
+
+* Unix timestamp
+* Nanosecond timestamp
+* Priority
+* Source host
+* Body
+* Metadata table with an arbitrary number of attribute value pairs.
+
+All events are guaranteed to have all of these elements.  However, the
+body may have zero length, and the metadata table can be empty.
+
+The Unix timestamp is measured in milliseconds and is Unix time stamp from the 
+source machine.  The nanosecond timestamp is machine specific nanosecond 
+counter also from the source machine.  It is safe to assume that the nanotime 
+from a machine is monotonically increasing -- i.e. if event A has a larger 
+nanotime than event B from the same machine, event A was initially received 
+before event B.
+
+Currently the priority of a message can have one of 6 values: TRACE, DEBUG, 
+INFO, WARN, ERROR, or FATAL.  These values are often provided by logging 
+systems such as syslog or log4j.  
+
+The source host is the name of the machine or the IP (whatever hostname call 
+returns).  
+
+The body is the raw log entry body.  The default is to truncate the body to a 
+maximum of 32KB per event.  This is a configurable value and can be changed by 
+modifying the 'flume.event.max.size.bytes' property.
+
+Finally there is the metadata table which is a map from a string attribute 
+name to an arbitrary array of bytes.  This allows for custom bucketing 
+attributes and will be described in more depth in the Advanced Usage section 
+of this guide.
+
+=== Output Bucketing
+
+You can control the output of events to particular directories or files based 
+on the values of an event's fields.  To enable this, you provide an escaping 
+mechanism that outputs data to a particular path.  
+
+For example, here is an output spec:
+----
+collectorSink("hdfs://namenode/flume/webdata/%H00/", "%{host}-")
+----
+
+// TODO describe the actual tags values when they are solidified
+
+The first argument is the directory where data is to be written.  The second 
+is a filename prefix where events are written.  Suppose you get an event from a 
+machine called server1 generated at time 18:58.  The events would get written 
+to HDFS with namenode namenode, in a directory called /flume/webdata/1800/, 
+with files named server1-xxx where xxx is some extra data for unique file 
+names.
+
+What happened here?  Flume replaced the '%H' with a string that represent the 
+hour of the timestamp found in the event's data. Likewise, the '%o' was 
+replace with the hostname field from the event.
+
+What happens if the server1's message had been delayed and the message wasn't 
+sent downstream until 19:05?  Since the value of the timestamp on the event 
+was during the 18:00 hour, the event would be written into that directory.  
+
+.Event data escape sequences
+[horizontal] %\{host\} ::  host
+%\{nanos\} :: nanos
+%\{priority\} :: priority string
+%\{body\} :: body
+%% :: a '%' character.
+%t :: Unix time in millis
+
+Because bucketing by date is a frequently-requested feature, there are escape 
+sequences for finer control of date values that allow you to bucket data based 
+on date.
+
+
+Here is another output spec:
+---- 
+collectorSink("hdfs://namenode/flume/webdata/%Y-%m-%d/%H00/", "web-")
+----
+
+This would create directories for each day, each with a subdirectory for each 
+hour with filenames prefixed "web-".
+
+.Fine grained escape sequences date and times
+[horizontal]
+%a :: locale's short weekday name (Mon, Tue, ...)
+%A :: locale's full weekday name (Monday, Tuesday, ...)
+%b :: locale's short month name (Jan, Feb,...)
+%B :: locale's long month name (January, February,...)
+%c :: locale's date and time (Thu Mar  3 23:05:25 2005)
+%d :: day of month (01)
+%D :: date; same as %m/%d/%y
+%H :: hour (00..23)
+%I :: hour (01..12)
+%j :: day of year (001..366)
+%k :: hour ( 0..23)
+%l :: hour ( 1..12)
+%m :: month (01..12)
+%M :: minute (00..59)
+%P :: locale's equivalent of am or pm
+%s :: seconds since 1970-01-01 00:00:00 UTC
+%S :: second (00..60)
+%y :: last two digits of year (00..99)
+%Y :: year (2010)
+%z :: +hhmm numeric timezone (for example, -0400)
+
+
+=== Output Format
+
+Now that you have control of where files go, this section describes how you can 
+control the output format of data.  Currently, this is set via the 
+'flume.collector.output.format' property set in the flume-site.xml file.  The output 
+formats are:
+
+.Output formats
+[horizontal]
+avro :: Avro Native file format.  Default currently is uncompressed.
+avrodata :: Binary encoded data written in the avro binary format.
+avrojson :: JSON encoded data generated by avro.
+default :: a debugging format.
+json :: JSON encoded data.
+log4j :: a log4j pattern similar to that used by CDH output pattern.
+raw :: Event body only.  This is most similar to copying a file but
+does not preserve any uniqifying metadata like host/timestamp/nanos.
+syslog :: a syslog like text output format.
+
+----
+<property>
+  <name>flume.collector.output.format</name>
+  <value>avrojson</value>
+  <description>This is the output format for the data written to the
+  collector.  There are several formats available:
+    avro - Avro Native file format.  Default currently is uncompressed.
+    avrodata - this outputs data as an avro binary encoded data
+    avrojson - this outputs data as json encoded by avro
+    debug - this is a format for debugging
+    json - this outputs data as json
+    log4j - outputs events in a pattern similar to Hadoop's log4j pattern
+    raw - Event body only.  This is most similar to copying a file but
+      does not preserve any uniqifying metadata like host/timestamp/nanos.
+    syslog - outputs events in a syslog-like format
+  </description>
+</property>  
+----
+
+
+=== Small Files Compared to High Latency
+
+For all versions Hadoop's file system that are earlier than 0.20.x,
+HDFS has write-once read-many semantics. Thus, the only way to
+reliably flush an HDFS file is to close the file.  Moreover, once a
+file is closed, no new data can be appended to the file.  This
+presents a tension between getting data written quickly to HDFS and
+potentially having many small files (which is a potential scalability
+bottleneck of HDFS).
+
+On one side, to minimize the load and data stored throughput the system, 
+ideally one would flush data to HDFS as soon as it arrives. Flushing 
+frequently is in conflict with efficiently storing data to HDFS because this 
+could result in many small files, which eventually will stress an HDFS 
+namenode.  A compromise is to pick a reasonable trigger that has a collector 
+close "reasonably-sized" files (ideally larger than a single HDFS block, 64MB 
+by default).  
+
+When Flume is deployed at a scale where data collection volumes are small, it 
+may take a long time to reach the ideal minimum file size (a block size, 
+typically 64MB).  For example, if a single web server produces 10k of logs a 
+second (approx. 100 hit logs/s at 100B per log), it will take about 2 hours 
+(6400 seconds) before an ideal file size can reached.  
+
+In these situations, lean towards having more small files.  Small files 
+cause a few problems downstream.  These include potential scaling limitations 
+of Hadoop's HDFS, and performance penalties when using MapReduce's default 
+input processing mechanisms within Hadoop.
+
+The following sections describe two mechanisms to mitigate these  
+potential problems:
+
+* Rolling up many small data files into larger batches 
+* Using a CombinedFileInputFormat
+
+This particular problem becomes less of an issue when the scale of logging 
+goes up.  If a hundred machines were generating the same amount of logs, you 
+would reach reasonable files sizes every 64 seconds.
+
+Future versions of Hadoop will mitigate this problem by providing a flush/sync 
+operation for currently open HDFS files (patch is already slated for 
+Hadoop HDFS 0.21.x).
+
+
+== Compression for files written to HDFS.
+
+Flume supports basic compression for all log files that are written to
+HDFS.  Compressed files are automatically suffixed with an extension
+and follow the same naming format + directory structure as regular log
+files.
+
+If GzipCodec is selected, ".gz" is appended to the file name, if
+BZip2Codec is selected, ".bz2" is appended.
+
+----
+  <property>
+    <name>flume.collector.dfs.compress.codec</name>
+    <value>None</value>
+    <description>Writes formatted data compressed in specified codec to
+    dfs. Value is None, GzipCodec, DefaultCodec (deflate), BZip2Codec,
+    or any other Codec Hadoop is aware of </description>
+  </property>
+----
+
+/////////////// 
+START OF COMMENTED OUT SECTION
+
+==== Rolling up Date into Larger Batches
+
+TODO (jon) Henry writes up this section.
+
+=== Integrating Flume Data Directly with Map Reduce
+
+MapReduce by default assigns a map job to each individual file specified as 
+input.  In the situation with many small files, one can use the 
+CombineFileInputFormat as the input format class of a MapReduce job.
+
+More details can be found here:
+
+http://hadoop.apache.org/common/docs/current/api/org/apache/hadoop/mapred/lib/
+CombineFileInputFormat.html
+
+Here's an example of code snippet in a java MapReduce jobs submission.
+
+---- 
+... 
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
+...
+    JobConf job = new JobConf();
+    job.setInputFormat(CombineFileInputFormat.class);
+
+----
+
+// TODO (jon) this is not done yet.
+
+
+=== Integrating Flume Data with Map Reduce Streaming
+
+// does the streaming command allow for user specified inputformats? ----
+
+bin/hadoop jar build/hadoop-streaming.jar --inputformat org.apache.hadoop.mapred.lib.CombineFileInputFormat ... 
+----
+
+
+=== Integrating Using Hive with Flume Data (experimental)
+
+Use avrojson output format, use Hive Avro serde (in Hive 0.x.x).
+
+Use other format and make regexes to parse data.
+
+END OF COMMENTED OUT SECTION
+//////////////
+
+
+
diff --git a/flume-docs/src/docs/UserGuide/VersionHistory b/flume-docs/src/docs/UserGuide/VersionHistory
new file mode 100644
index 0000000..dfdcf40
--- /dev/null
+++ b/flume-docs/src/docs/UserGuide/VersionHistory
@@ -0,0 +1,65 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Versions
+
+=== History
+
+v0.9.3 2/4/11:: Multi-master functional.  Microsoft Windows support.
+Metrics framework including json support. Updated Source/Sink API.
+Improved robustness of non-translated agent chains and multiple
+logical node configurations.  Updated configuration languages
+cconstructs.  Usability improvements.
+
+v0.9.2 11/15/10:: Support for all Hadoop supported compression codecs.
+Avro RPC support.  Flexible throughput throttling.  Useful error
+messages in shell. Upgrades to Thrift RPC support.  Performance and
+robustness improvements in tail and exec.
+
+v0.9.1u1 10/12/10:: Added write support for Kerberized HDFS. Flume
+cookbook.
+
+v0.9.1 8/9/10:: Improved error messages and visibility of property
+configuration values.  First external contributions.  Fixed
+reconfiguration hangs.  Improved implementing plugins documentation.
+Updated scribe and syslog support.  Compression on output files.
+
+v0.9 6/29/10 :: metrics and reporting framework, logical nodes+logical
+names abstraction, wal/dfo isolation by flow, transformation-based
+high level sinks.  Open source and initial push to github.
+
+v0.3 3/31/10 :: ZK based master/multi-master, automatic failovers for
+data and control planes.  flume shell. deb/rpm packaging.
+
+v0.2 1/21/10 :: Different reliability modes: WAL 2.0, DFO, Best
+effort. Output file escaping/bucketing.  Proliferation of many sink
+and decorators.
+
+v0.1 11/23/09 :: First installation deployment, users tests.
+
+v0.0 9/21/09 :: First cut with current architecture (centralized
+master, configuration language, web interface.)  First version of WAL.
+Simple visualizations, samplers, thrift based rpc.
+
+pre history 7/21/09 :: First commit.  design, experimental
+implementations.  Initial implementation had individual agent and
+collector programs, watchdog
+
+
+
diff --git a/flume-docs/src/docs/Windows/Windows b/flume-docs/src/docs/Windows/Windows
new file mode 100644
index 0000000..1a364c2
--- /dev/null
+++ b/flume-docs/src/docs/Windows/Windows
@@ -0,0 +1,208 @@
+Installing Flume Nodes on Windows
+=================================
+flume-dev@cloudera.org
+
+// version number comes from command line
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Flume on Microsoft Windows-based Machines
+
+
+image:flume-logo.png[]
+
+Flume v0.9.3 is the first version of Flume to support Windows-based
+Flume nodes.  It has been tested on Window Server 2003 SP2 and Windows
+XP SP3 machines and can be used to collect logs generated from Windows
+applications such as Microsoft IIS.
+
+Requirements:
+
+* Sun/Oracle 32-bit Windows JRE v1.6+
+
+NOTE: We require using the 32-bit JVM even on 64 bit machines because
+of lower memory footprint.
+
+=== Installation
+
+Flume for Windows is installed using an executable installer generated
+by Nullsoft Scriptable Install System (NSIS).  It provides a simple
+wizard which presents basic options for a node's configuration
+settings.  It will install the software and optionally start a Flume
+node as a Windows service (daemon).
+
+NOTE: Before reinstalling the Flume on Windows service, use the
+Windows Services Management Console to stop the +flumenode.exe+
+service.
+
+When you start the Flume for Windows installer, the following dialog
+box appears.  Click "Next".
+
+image:flume-setup.png[]
+
+Read the license agreement and then click "I Agree".
+
+image:flume-setup-2-license.png[]
+
+In the drop-down menu in the following dialog box, specify whether you
+want to install the Flume node core software, or install the Flume
+node core software and install it as a Windows service.  Installing as
+a service will add the Flume node service that is started when the
+machine is rebooted.  
+
+Select the components you want to install and then click "Next" to
+continue.
+
+image:flume-setup-3-components.png[]
+
+Specify the Flume node's master location. The default is "localhost"
+which specifies the current machine.  To specify a different machine,
+enter either its IP address or machine name.  Also, specify the TCP
+port for node-to-master communications.  The default port is
+35862. Click "Next" to continue.
+
+image:flume-setup-4-master.png[]
+
+Next the installer attempts to automatically detect the location of an
+installed Java Runtime Environment (JRE).  If the JRE is not found,
+manually specify the path.  Click "Next".
+
+WARNING: Currently the installer requires a 32-bit JVM.  Using a
+64-bit JVM is not supported yet.
+
+image:flume-setup-5-jre.png[]
+
+Specify the directory where you want to install the Flume for Windows
+components, and then click "Install".
+
+image:flume-setup-6-installdir.png[]
+
+After installation is complete, the following dialog box appears.
+Click "Finish".
+
+image:flume-setup-7-complete.png[]
+
+You can confirm that the Flume node is up by:
+
+* Opening a browser to http://localhost:35862 
+* Checking that the +flumenode.exe+ service is started in the Windows
+  Task Manager or in the Windows Services Management Console
+* Seeing the following tray icon:
+
+image:flume-tray-icon.png[]
+
+=== Administering Windows Flume Nodes
+
+Installation by default will set up the Flume node to automatically
+start when the Windows machine is booted.  You can manually start and
+stop the Flume node service by using:
+
+* The tray icon applet
+* The +sc+ (service controller) windows command line program 
+* The Windows Service Management Console
+
+You can use the tray icon applet to start, stop, or restart the Flume
+node service. You can access the commands by right clicking on the
+tray icon and displaying the following menu:
+
+image:flume-tray-menu.png[]
+
+Click "Configure..." to change the Flume node service configuration
+settings, such as JVM parameters and the Restart-on-reboot mode.
+
+NOTE: Flume relies on Apache Common Daemon's +prunmgr.exe+ and
++prunsvr.exe+ programs.  These have been renamed to +flumenode.exe+
+and +flumenodew.exe+.  The first runs the Flume node as a Windows
+Service, and the second provides the tray icon configuration
+interface.
+
+Alternately, you can start and stop the Flume node service by typing
+the following commands (as the Administrator) at the Windows command
+prompt:
+
+----
+C:\> sc FlumeNode start
+C:\> sc FlumeNode stop
+----
+
+Run +sc+ for more details and options.
+
+Unfortunately, the +install+, +delete+, and +create+ methods do not
+always work.  To manually install and uninstall the Flume node
+service, run the following scripts from the command prompt (as
+Administrator) at the Windows command prompt:
+
+----
+%FLUME_HOME%\bin\> flumenode-service.bat install
+
+$FLUME_HOME%\bin\> flumenode-service.bat remove
+----
+
+WARNING: Uninstalling Flume node may not be completed cleanly.  You
+may need to stop +flumenode.exe+ and +flumenodew.exe+ using the Task
+Manager.
+
+=== Default Directories.
+
+The installation directory where the Flume node programs are installed
+is called +%FLUME_HOME%+.  By default this is +C:\Program
+Files\Cloudera\Flume 0.9.3\+.
+
+[width="50%",options="header"]
+|=================================================================================
+|Directory          |Purpose
+|%FLUME_HOME%\logs  |Directory where Flume node logs are saved.
+|%FLUME_HOME%\conf  |Directory where flume-site.xml and flume-conf.xml are located
+|=================================================================================
+
+The +FLUME_CONF_DIR+ environment variable is respected -- you can keep
+xml configs in a separate directory pointed to by this variable for
+customized configurations.
+
+=== Building Flume for Windows
+
+To build Flume for Windows, you need:
+
+* Sun/Oracle JDK v1.6.x+
+* cygwin (http://www.cygwin.com/)
+* ant (http://ant.apache.org/bindownload.cgi, .zip version)
+* NSIS (we are using v2.46, http://nsis.sourceforge.net/Main_Page)
+
+With cygwin the normal Unix build process can be used.  For example,
+assuming you are in the Flume project directory, type this command:
+
+----
+~/flume$ ant 
+-----
+
+To generate the Flume installer executable, run the following special
+ant filter to inject the proper build-specific versions information:
+
+----
+~/flume$ ant winstall-filter
+----
+
+You can build the Flume for Windows installer manually. Start NSIS,
+and then click "Compile NSI script". In the dialog box that opens,
+select "File | Load script…" from the menu. Select flume/flume.nsi,
+and then click "Open". This will compile the script and generate the
+installer, +flume-installer.exe+. This file should be located in the
+main Flume project directory.
+
+
diff --git a/flume-docs/src/docs/Windows/flume-logo.png b/flume-docs/src/docs/Windows/flume-logo.png
new file mode 100755
index 0000000000000000000000000000000000000000..bd119fa07886edf5781ae136b8c7473baedcf73e
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup-2-license.png b/flume-docs/src/docs/Windows/flume-setup-2-license.png
new file mode 100755
index 0000000000000000000000000000000000000000..b6ad9143b8f41a0c4a21dba87e859f47ca5abe36
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup-3-components.png b/flume-docs/src/docs/Windows/flume-setup-3-components.png
new file mode 100755
index 0000000000000000000000000000000000000000..1010f754de68fbcc73169212a18a4431e40b95ca
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup-4-master.png b/flume-docs/src/docs/Windows/flume-setup-4-master.png
new file mode 100755
index 0000000000000000000000000000000000000000..72fb600a71c59c8433b2b24c7e28d388a4e9f5de
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup-5-jre.png b/flume-docs/src/docs/Windows/flume-setup-5-jre.png
new file mode 100755
index 0000000000000000000000000000000000000000..29d37be7c4663f6ca4aa8bb58e321221a43eb592
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup-6-installdir.png b/flume-docs/src/docs/Windows/flume-setup-6-installdir.png
new file mode 100755
index 0000000000000000000000000000000000000000..289e4f81f9059e78f6d45b390b73a43eccf74783
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup-7-complete.png b/flume-docs/src/docs/Windows/flume-setup-7-complete.png
new file mode 100755
index 0000000000000000000000000000000000000000..9ad3128545e44fcc6a8e9c87921c885d0f361725
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-setup.png b/flume-docs/src/docs/Windows/flume-setup.png
new file mode 100755
index 0000000000000000000000000000000000000000..4b75eb85c40a54d09c99e575bc6cc4c4e96ab1d1
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-tray-icon.png b/flume-docs/src/docs/Windows/flume-tray-icon.png
new file mode 100755
index 0000000000000000000000000000000000000000..46b023bc7bbf2516320e2247bc654b8e22198622
GIT binary patch
diff --git a/flume-docs/src/docs/Windows/flume-tray-menu.png b/flume-docs/src/docs/Windows/flume-tray-menu.png
new file mode 100755
index 0000000000000000000000000000000000000000..8dc5f757c34096a8704cd8808de413998ed3891a
GIT binary patch
diff --git a/flume-docs/src/docs/cookbook/Apache b/flume-docs/src/docs/cookbook/Apache
new file mode 100644
index 0000000..21c60bb
--- /dev/null
+++ b/flume-docs/src/docs/cookbook/Apache
@@ -0,0 +1,178 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Using Flume Agents for Apache 2.x Web Server Logging
+
+To connect Flume to Apache 2.x servers, you will need to
+
+# Configure web log file permissions 
+
+# Tail the web logs or use piped logs to enable Flume to get data from
+  the web server.
+
+This section will step through basic setup on default Ubuntu Lucid and
+default CentOS 5.5 installations.  Then it will describe various ways
+of integrating Flume.
+
+=== If you are using CentOS / Red Hat Apache servers
+
+By default, CentOS's Apache writes weblogs to files owned by +root+
+and in group +adm+ in 0644 (rw-r--r--) mode.  Flume is run as the
++flume+ user, so the Flume node is able to read the logs.
+
+Apache on CentOS/Red Hat servers defaults to writing logs to two
+files:
+
+----
+/var/log/httpd/access_log
+/var/log/httpd/error_log
+----
+
+The simplest way to gather data from these files is to tail the files
+by configuring Flume nodes to use Flume's +tail+ source:
+
+----
+tail("/var/log/httpd/access_log") 
+tail("/var/log/httpd/error_log")
+----
+
+=== If you are using Ubuntu servers Apache servers
+
+By default, Ubuntu writes weblogs to files owned by +root+ and in
+group +adm+ in 0640 (rw-r-----) mode.  Flume is run as the +flume+
+user and by default will *not* be able to tread the files.  One
+approach to allow the +flume+ user to read the files is to add it to
+the +adm+ group.
+
+Apache servers on Ubuntu defaults to writing logs to three files:
+
+----
+/var/log/apache2/access.log
+/var/log/apache2/error.log
+/var/log/apache2/other_vhosts_access.log
+----
+
+The simplest way to gather data from these files is by configuring
+Flume nodes to use Flume's +tail+ source:
+
+----
+tail("/var/log/apache2/access.log") 
+tail("/var/log/apache2/error.log")
+tail("/var/log/apache2/other_vhosts_access.log")
+----
+
+
+=== Getting log entries from Piped Log files
+
+The Apache 2.x's documentation
+(http://httpd.apache.org/docs/2.2/logs.html) describes using piped
+logfile with the +CustomLog+ directive.  Their example uses
++rotatelogs+ to periodically write data to new files with a given
+prefix.  Here are some example directives that could be in the
++httpd.conf+/+apache2.conf+ file.
+
+----
+LogFormat "%h %l %u %t \"%r\" %>s %b" common
+CustomLog "|/usr/sbin/rotatelogs /var/log/apache2/foo_access_log 3600" common
+----
+
+TIP: In Ubuntu Lucid these directives are in
++/etc/apache2/sites-available/default+. In CentOS 5.5, these
+directives are in +/etc/httpd/conf/httpd.conf+.
+
+These directives configure Apache to write log files in
++/var/log/apache2/foo_access_log.xxxxx+ every hour (3600 seconds)
+using the "common" log format.
+
+You can use Flume's +tailDir+ source to read all files without
+modifying the Apache settings:
+
+----
+tailDir("/var/log/apache2/", "foo_access_log.*")
+----
+
+The first argument is the directory, and then the second is a regex
+that should match against the file name.  +tailDir+ will watch the dir
+and tail all files that have matching file names.
+
+=== Using piped logs
+
+Instead of writing data to disk and then having Flume read it, you can
+have Flume ingest data directly from Apache.  To do so, modify the web
+server's parameters and use its piped log feature by adding some
+directives to Apache's configuration:
+
+----
+CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentBESink(\"collector\");\'" common
+----
+
+----
+CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentDFOSink(\"collector\");\'" common
+----
+
+WARNING: By default, CentOS does not have the Java required by the
+Flume node in user +root+'s path.  You can use +alternatives+ to
+create a managed symlink in +/usr/bin/+ for the java executable.
+
+Using piped logs can be more efficient, but is riskier because Flume
+can deliver messages without saving on disk.  Doing this, however,
+increases the probability of event loss.  From a security point of
+view, this Flume node instance runs as Apache's user which is often
++root+ according to the Apache manual.
+
+NOTE: You could configure the one-shot mode node to deliver data
+directly to a collector.  This can only be done at the best effort or
+disk-failover level.
+
+The prior examples use Flume nodes in one-shot mode which runs without
+contacting a master.  Unfortunately, it means that one-shot mode
+cannot directly use the automatic chains or the end-to-end (E2E)
+reliability mode.  This is because the automatic chains are generated
+by the master and because E2E mode currently delivers acknowledgements
+through the master.  
+
+However, you can have a one-shot Flume node deliver data to a Flume
+local node daemon where the reliable E2E mode can be used.  In this
+setup we would have the following Apache directive:
+
+----
+CustomLog "|flume node_nowatch -1 -s -n apache -c \'apache:console|agentBESink(\"localhost\", 12345);\'" common
+----
+
+Then you can have a Flume node setup to listen with the following
+configuration:
+
+----
+node : rpcSource(12345) | agentE2ESink("collector");
+----
+
+Since this daemon node is connected to the master, it can use the
+auto*Chains.
+
+----
+node : rpcSource(12345) | autoE2EChain;
+----
+
+NOTE: End-to-end mode attempts to ensure the deliver of the data that
+enters the E2E sink.  In this one-shot-node to reliable-node scenario,
+data is not safe it gets to the E2E sink.  However, since this is a
+local connection, it should only fail when the machine or processes
+fails.  The one-shot node can be set to disk failover (DFO) mode in
+order to reduce the chance of message loss if the daemon node's
+configuration changes.
diff --git a/flume-docs/src/docs/cookbook/Cookbook b/flume-docs/src/docs/cookbook/Cookbook
new file mode 100644
index 0000000..9a2b86a
--- /dev/null
+++ b/flume-docs/src/docs/cookbook/Cookbook
@@ -0,0 +1,65 @@
+Flume Cookbook
+===============
+flume-dev@cloudera.org
+
+// version number comes from command line.
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+// Upgrading to use asciidoc 8.4.4
+// Build using 'asciidoc -a toc FullGuide'
+
+== Introduction
+
+Flume is a distributed, reliable, and available service for
+efficiently collecting, aggregating, and moving large amounts of log
+data.  Being highly configurable and very extensible means that there
+are many options and thus many decisions that need to be made by an
+operator.  This document is a "cookbook" with "recipes" for getting
+Flume up and running quickly without being overwhelmed with all the
+details.
+
+This document starts with some basic tips on experimenting with Flume
+nodes, and then three stories about setting up agents for different
+kinds of sources.  Finally we'll do a quick setup for collectors and
+how to troubleshoot distributed deployments.
+
+* Basic debugging with one-shot Flume nodes
+* Flume agents for Apache web server logging
+* Flume agents for syslog / syslog-ng logging
+* Flume agents for scribe logging
+// * Flume agents for logging via log4j
+// * Configuring nodes on node startup
+// * Small Deployments.
+
+include::Testing[]
+include::Apache[]
+include::Syslog[]
+include::Scribe[]
+// include::Log4J[]
+// include::ConfigOnStart[]
+// include::SmallDeploy[]
+
+
+     ______
+    / ___//_  ______  ____
+   / /_/ / / / /    \/ __/
+  / __/ / /_/ / / / / __/
+ / / /_/\____/_/_/_/\__/
+/_/ Distributed Log Collection.
diff --git a/flume-docs/src/docs/cookbook/Scribe b/flume-docs/src/docs/cookbook/Scribe
new file mode 100644
index 0000000..6b892f5
--- /dev/null
+++ b/flume-docs/src/docs/cookbook/Scribe
@@ -0,0 +1,60 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Logging Scribe Events to a Flume Agent
+
+Flume can emulate a downstream node for applications that log using
+Scribe.  Scribe is Facebook's open source log aggregation framework.
+It has a simple API and uses Thrift as its core network transport
+mechanism.  Flume uses the same Thrift IDL file and can listen for
+data provided by Scribe sources.
+
+Scribe comes with an example application called +scribe_cat+.  It acts
+like the unix +cat+ program but sends data to a downstream host with a
+specified "category".  Scribe by default uses TCP port 1463.
+
+You can configure a Flume node to listen for incoming Scribe traffic
+by creating a logical node that uses the +scribe+ source.  We can then
+assign an arbitrary sink to the node.  In the example below, the
+Scribe nodes receives events, send its events to both the console and
+an automatically-assigned end-to-end agent which delivers the events
+downstream to its collector pools.
+
+----
+scribe: scribe | [console, autoE2EChain];
+----
+
+Assuming that this node communicates with the master, and that the
+node is on +localhost+, we can use the +scribe_cat+ program to send
+data to Flume.
+
+----
+$ echo "foo" | scribe_cat localhost testcategory
+----
+
+When the Scribe source accepts the Scribe events, it converts Scribe's
+category information into a new Flume event metadata entry, and then
+delivers the event to its sinks.  Since Scribe does not include time
+metadata, the timestamp of the created Flume event will be the arrival
+time of the Scribe event.
+
+
+
+
+
diff --git a/flume-docs/src/docs/cookbook/Syslog b/flume-docs/src/docs/cookbook/Syslog
new file mode 100644
index 0000000..948fc98
--- /dev/null
+++ b/flume-docs/src/docs/cookbook/Syslog
@@ -0,0 +1,245 @@
+
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Flume Agents for Syslog data
+
++syslog+ is the standard unix single machine logging service.  Events
+are generally emitted as lines with a time stamp, "facility" type,
+priority, and message.  Syslog can be configured to send data to
+remote destinations.  The default syslog remote delivery was
+originally designed to provide best effort delivery service.  Today,
+there are several more advanced syslog services that deliver messages
+with improved reliability (TCP connections with memory buffering on
+failure).  The reliability guarantees however are one hop and weaker
+than Flume's more reliable delivery mechanism.
+
+This section describes collecting syslog data using two methods.  The
+first part describes a file tailing approach.  The latter parts
+describe syslog system configuration guidance that enables directly
+feeding Flume's +syslog*+ sources.
+
+=== Tailing files
+
+The quickest way to record syslog messages is to tail syslog generated
+log files.  These files generally live in +/var/log+.
+
+Some examples include:
+----
+/var/log/auth.log
+/var/log/messages
+/var/log/syslog
+/var/log/user.log
+----
+
+These files could be tailed by Flume nodes with tail sources:
+
+----
+tail("/var/log/auth.log")
+tail("/var/log/messages")
+tail("/var/log/syslog")
+tail("/var/log/user.log")
+----
+
+Depending on your system configuration, there may be permissions
+issues when accessing these files from the Flume node process.
+
+NOTE: Red Hat/CentOS systems default to writing log files owned by
+root, in group root, and with 0600 (-rw-------) permissions. Flume
+could be run as root, but this is not advised because Flume can be
+remotely configured to execute arbitrary programs.
+
+NOTE: Ubuntu systems default to writing logs files owned by syslog, in
+group adm, and with 0640 (-rw-r-----) permissions.  By adding the user
+"flume" to group "adm", a Flume node running user "flume" should be
+able to read the syslog generated files.
+
+NOTE: When tailing files, the time when the event is read is used as the
+time stamp.
+
+=== Delivering Syslog events via sockets
+
+The original syslog listens to the +/dev/log+ named pipe, and can be
+configured to listen on UDP port
+514. (http://tools.ietf.org/search/rfc5424). More advanced versions
+(rsyslog, syslog-ng) can send and recieve over TCP and may do
+in-memory queuing/buffering. For example, syslog-ng and rsyslog can
+optionally use the default UDP port 514 or use TCP port 514 for better
+recovery options.
+
+NOTE: By default only superusers can listen on on UDP/TCP ports 514.
+Unix systems usually only allow ports <1024 to be bound by
+superusers. While Flume can run as superuser, from a security stance
+this is not advised.  The examples provide directions to route to the
+user-bindable port 5140.
+
+For debugging syslog configurations, you can just use 'flume dump'
+with syslog sources.  This command outputs received syslog data to the
+console.  To test if syslog data is coming in to the proper port you
+can run this command from the command line:
+
+----
+$ flume dump 'syslogUdp(5140)' 
+----
+
+This will dump all incoming events to the console.
+
+If you are satisfied with your connection, you can have a Flume node
+run on the machine configure its sink for the reliability level you
+desire.
+
+Using a +syslog*+ Flume source will save the entire line of event
+data, use the timestamp found in the original data, extract a +host+,
+and attempt to extract a service from the syslog line.  All of these
+map to a Flume event's fields except for +service+ so this is added as
+extra metadata field to each event (this is a convention with
+syslog defined in RFC).
+
+So, a syslog entry whose body is this:
+
+----
+Sep 14 07:57:24 blitzwing dhclient: bound to 192.168.126.212 -- renewal in 710 seconds.
+----
+
+will have the Flume event body:
+
+----
+Sep 14 07:57:24 blitzwing dhclient: bound to 192.168.126.212 -- renewal in 710 seconds.
+----
+
+The event will also translated the "Sep 14 07:57:24" date+time data so
+that it will be bucketable.  Since this date does not have a year, it
+assumes the current year and since it has no timezone it assumes the
+local timezone.  The host field should be "blitzwing", and the
+optional "service" metadata field will contain "dhclient".
+
+====  Configuring +syslogd+
+
+The original syslog is +syslogd+.  It is configured by an
++/etc/syslog.conf+ file.  Its format is fairly simple.
+
+Syslog recieves messages and then sends to out to different facilities
+that have associated names
+(http://tools.ietf.org/search/rfc5424#section-6.2).
+
+The +/etc/syslog.conf+ file essentially contains lists of facilities
+and "actions".  These "actions" are destinations such as regular
+files, but can also be named pipes, consoles, or remote machines.  One
+can specify a remote machine by prefixing an '@' symbol in front the
+destination host machine.  If no port is specified, events are sent
+via UDP port 514.
+
+The example below specifies delivery to machine localhost on port
+5140.
+
+----
+user.*     @localhost:5140 
+----
+
+A Flume node daemon running on this machine would have a +syslogUdp+
+source listening for new log data.
+
+----
+host-syslog : syslogUdp(5140) | autoE2EChain ;
+----
+
+==== Configuring +rsyslog+ 
+
++rsyslog+ is a more advanced drop-in replacement for syslog and the
+default syslog system used by Ubuntu systems.  It supports basic
+filtering, best effort delivery, and queuing for handling one-hop
+downstream failures.
+
++rsyslog+ actually extends the syslog configuration file format.
+Similar to regular +syslogd+ you can send data to a remote machine on
+listening on UDP port 514 (standard syslog port).
+
+----
+*.*   @remotehost
+----
+
+Moreover, +rsyslog+ also allows you to use the more reliable TCP
+protocol to send data to a remote host listening on TCP port 514.  In
++rsyslog+ configurations, an '@@' prefix dictates the use of TCP.
+
+----
+*.*  @@remotehost
+----
+
+Similarly, you can also append a suffix port number to have it deliver
+to a particular port.  In this example, events are delivered to
+localhost TCP port 5140.
+
+----
+*.*  @@localhost:5140
+----
+
+Assuming you have a Flume node daemon running on the local host, you
+can capture syslog data by adding a logical node with the following
+configuration:
+
+----
+host-syslog : syslogTcp(5140) | autoE2EChain ; 
+----
+
+////
+
+TODO: (this requires new FileReaderSource)
+
+You can also log data to a named pipe that flume can listen on.
+
+Named pipes
+----
+*.*  |/dev/flume
+----
+
+////
+
+==== Configuring +syslog-ng+
+
+Syslog-ng is another common replacement for the default syslog logging
+system.  Syslog-ng has a different configuration file format but
+essentially gives the operator the ability to send syslog data from
+different facilities to different remote destinations.  TCP or UDP can
+be used.
+
+Here is an example of modifications to a +syslog-ng.conf+ (often found
+in +/etc/syslog-ng/+) file.
+
+----
+## set up logging to loghost (which is flume) 
+destination loghost { 
+        tcp("localhost" port(5140)); 
+}; 
+
+# send everything to loghost, too 
+log { 
+        source(src); 
+        destination(loghost); 
+}; 
+----
+
+Assuming you have a Flume node daemon running on the local host, you
+can capture syslog data by adding a logical node with the following
+configuration:
+
+----
+host-syslog : syslogTcp(5140) | autoE2EChain ; 
+----
+
diff --git a/flume-docs/src/docs/cookbook/Testing b/flume-docs/src/docs/cookbook/Testing
new file mode 100644
index 0000000..955487c
--- /dev/null
+++ b/flume-docs/src/docs/cookbook/Testing
@@ -0,0 +1,173 @@
+
+////////////////////
+Licensed to Cloudera, Inc. under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  Cloudera, Inc. licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////////////////////
+
+== Trying out Flume sources and sinks
+
+First this section will describe some basic tips for testing sources,
+sinks and logical nodes.
+
+=== Testing sources
+
+Testing sources is a straightforward process.  The +flume+ script has
+a +dump+ command that allows you test data source configuration by
+displaying data at the console.
+
+----
+flume dump source
+----
+
+NOTE: +source+ needs to be a single command line argument, so you may
+need to add 'quotes' or "quotes" to the argument if it has quotes or
+parenthesis in them.  Using single quotes allow you to use unescaped
+double quotes in the configuration argument. (ex: `'text("/tmp/foo")'`
+or `"text(\"/tmp/foo\")"`).
+
+Here are some simple examples to try:
+
+----
+$ flume dump console
+$ flume dump 'text("/path/to/file")'
+$ flume dump 'file("/path/to/file")'
+$ flume dump syslogTcp
+$ flume dump 'syslogUdp(5140)'
+$ flume dump 'tail("/path/to/file")'
+$ flume dump 'tailDir("path/to/dir", "fileregex")'
+$ flume dump 'rpcSource(12346)'
+----
+
+Under the covers, this dump command is actually running the +flume
+node_nowatch+ command with some extra command line parameters.
+
+----
+flume node_nowatch -1 -s -n dump -c "dump: $1 | console;"
+----
+
+Here's a summary of what the options mean.
+
+[horizontal]
++-1+ :: one shot execution.  This makes the node instance not use the
+heartbeating mechanism to get a config.
+
++-s+ :: starts the Flume node without starting the node's http status
+web server.  If the status web server is started, a Flume node's
+status server will keep the process alive even if in one-shot mode.
+If the -s flag is specified along with one-shot mode (-1), the Flume
+node will exit after all logical nodes complete.
+
++-c "node:src|snk;"+ :: Starts the node with the given configuration
+ definition.  NOTE: If not using -1, this will be invalidated
+ upon the first heartbeat to the master.
++-n node+ :: gives the node the physical name node.
+
+You can get info on all of the Flume node commands by using this command:
+
+----
+$ flume node -h
+----
+
+=== Testing sinks and decorators
+
+Now that you can test sources, there is only one more step necessary
+to test arbitrary sinks and decorators from the command line.  A sink
+requires data to consume so some common sources used to generate test
+data include synthetic datasets (+asciisynth+), the console
+(+console+), or files (+text+).
+
+For example, you can use a synthetic source to generate 1000 events,
+each of 100 "random" bytes data to the console. You could use a text
+source to read a file like /etc/services, or you could use the console
+as a source and interactively enter lines of text as events:
+
+----
+$ flume node_nowatch -1 -s -n dump -c 'dump: asciisynth(1000,100) | console;'
+$ flume node_nowatch -1 -s -n dump -c 'dump: text("/etc/services") | console;'
+$ flume node_nowatch -1 -s -n dump -c 'dump: console | console;'
+----
+
+You can also use decorators on the sinks you specify.  For example,
+you could rate limit the amount of data that pushed from the source to
+sink by inserting a delay decorator.  In this case, the 'delay'
+decorator waits 100ms before sending each synthesized event to the
+console.
+
+----
+$ flume node_nowatch -1 -s -n dump -c 'dump: asciisynth(1000,100) | { delay(100) => console};'
+----
+
+Using the command line, you can send events via the direct best-effort
+(BE) or disk-failover (DFO) agents.  The example below uses the
++console+ source so you can interactively generate data to send in BE
+and DFO mode to collectors.
+
+NOTE: Flume node_nowatch must be used when piping data in to a Flume
+node's console source.  The watchdog program does not forward stdin.
+
+----
+$ flume node_nowatch -1 -s -n dump -c 'dump: console | agentBESink("collectorHost");'
+$ flume node_nowatch -1 -s -n dump -c 'dump: console | agentDFOSink("collectorHost");'
+----
+
+WARNING: Since these nodes are executed with configurations entered at
+the command line and never contact the master, they cannot use the
+automatic chains or logical node translations.  Currently, the
+acknowledgements used in E2E mode go through the master piggy-backed
+on node-to-master heartbeats.  Since this mode does not heartbeat, E2E
+mode should not be used.
+
+Console sources are useful because we can pipe data into Flume
+directly.  The next example pipes data from a program into Flume which
+then delivers it.
+
+----
+$ <external process> | flume node_nowatch -1 -s -n foo -c 'foo:console|agentBESink("collector");'
+----
+
+Ideally, you could write data to a named pipe and just have Flume read
+data from a named pipe using +text+ or +tail+.  Unfortunately, this
+version of Flume's +text+ and +tail+ are not currently compatible with
+named pipes in a Linux environment.  However, you could pipe data to a
+Flume node listening on the stdin console:
+
+----
+$ tail -f namedpipe | flume node_nowatch -1 -s -n foo -c 'foo:console|agentBESink;'
+----
+
+Or you can use the exec source to get its output data:
+
+----
+$ flume node_nowatch -1 -s -n bar -c 'bar:exec("cat pipe")|agentBESink;' 
+----
+
+=== Monitoring nodes
+
+While outputting data to a console or to a logfile is an effective way
+to verify data transmission, Flume nodes provide a way to monitor the
+state of sources and sinks.  This can be done by looking at the node's
+report page.  By default, you can navigate your web browser to the
+nodes TCP 35862 port.  (http://node:35862/)
+
+This page shows counter information about all of the logical nodes on
+the physical node as well as some basic machine metric information.
+
+TIP: If you have multiple physical nodes on a single machine, there
+may be a port conflict.  If you have the auto-find port option on
+(+flume.node.http.autofindport+), the physical node will increment the
+port number until it finds a free port it can bind to.
+
+
diff --git a/flume-docs/src/docs/web/breadcrumbs.xsl b/flume-docs/src/docs/web/breadcrumbs.xsl
new file mode 100644
index 0000000..98d70ba
--- /dev/null
+++ b/flume-docs/src/docs/web/breadcrumbs.xsl
@@ -0,0 +1,47 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:exsl="http://exslt.org/common"
+                version="1.0"
+                exclude-result-prefixes="exsl">
+
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<xsl:template name="breadcrumbs">
+  <xsl:param name="this.node" select="."/>
+  <div class="breadcrumbs">
+    <xsl:for-each select="$this.node/ancestor::*">
+      <span class="breadcrumb-link">
+        <a>
+          <xsl:attribute name="href">
+            <xsl:call-template name="href.target">
+              <xsl:with-param name="object" select="."/>
+              <xsl:with-param name="context" select="$this.node"/>
+            </xsl:call-template>
+          </xsl:attribute>
+          <xsl:apply-templates select="." mode="title.markup"/>
+        </a>
+      </span>
+      <xsl:text> &gt; </xsl:text>
+    </xsl:for-each>
+    <!-- And display the current node, but not as a link -->
+    <span class="breadcrumb-node">
+      <xsl:apply-templates select="$this.node" mode="title.markup"/>
+    </span>
+  </div>
+</xsl:template>
+
+</xsl:stylesheet>
diff --git a/flume-docs/src/docs/web/docbook.css b/flume-docs/src/docs/web/docbook.css
new file mode 100644
index 0000000..a622c9f
--- /dev/null
+++ b/flume-docs/src/docs/web/docbook.css
@@ -0,0 +1,269 @@
+/*
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ *
+ *
+ * This file was provided to this project under these terms:
+ *
+ * Copyright (c) 2001, 2003 The FreeBSD Documentation Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: doc/share/misc/docbook.css,v 1.14 2008/11/21 07:28:34 keramida Exp $
+ */
+
+BODY ADDRESS {
+	line-height: 1.3;
+	margin: .6em 0;
+}
+
+BODY BLOCKQUOTE {
+	margin-top: .75em;
+	line-height: 1.5;
+	margin-bottom: .75em;
+}
+
+HTML BODY {
+	margin: 1em 4% 1em 4%;
+	line-height: 1.2;
+	font-family: verdana;
+}
+
+a {
+    text-decoration: none;
+    color: #1c84bf;
+}
+
+a:hover {
+    text-decoration: underline;
+}
+
+a:visited {
+    color: #597e93;
+}
+
+div.titlepage .title {
+	font-size: 26px;
+}
+
+.LEGALNOTICE {
+	font-size: small;
+	font-variant: small-caps;
+}
+
+BODY DIV {
+	margin: 0;
+}
+
+DL {
+	margin: .8em 0;
+	line-height: 1.2;
+}
+
+BODY FORM {
+	margin: .6em 0;
+}
+
+H1, H2, H3, H4, H5, H6,
+DIV.EXAMPLE P B,
+.QUESTION,
+DIV.TABLE P B,
+DIV.PROCEDURE P B {
+	color: #148BCF;
+}
+
+BODY H1, BODY H2, BODY H3, BODY H4, BODY H5, BODY H6 {
+	line-height: 1.3;
+	margin-left: 0;
+}
+
+BODY H1, H2, H3, H4, H5, H6 {
+	margin: 6px 0 0 0;
+}
+
+BODY HR {
+	border-width: 0 0 1px 0;
+	border-style: solid;
+	border-color: #0068BB;
+}
+
+BODY IMG.NAVHEADER {
+	margin: 0 0 0 -4%;
+}
+
+OL {
+	margin: 0 0 0 5%;
+	line-height: 1.2;
+}
+
+BODY PRE {
+	margin: .75em 0;
+	line-height: 1.0;
+	font-family: monospace;
+}
+
+BODY TD, BODY TH {
+	line-height: 1.2;
+}
+
+UL, BODY DIR, BODY MENU {
+	margin: 0 0 0 5%;
+	line-height: 1.2;
+}
+
+HTML {
+	margin: 0; 
+	padding: 0;
+}
+
+BODY P B.APPLICATION {
+	color: #000000;
+}
+
+.navfooter td, .literal, .FILENAME {
+	color: #007a00;
+	font-family: monospace;
+}
+
+
+.GUIMENU, .GUIMENUITEM, .GUISUBMENU,
+.GUILABEL, .INTERFACE,
+.SHORTCUT, .SHORTCUT .KEYCAP {
+	font-weight: bold;
+}
+
+.GUIBUTTON {
+	background-color: #CFCFCF;
+	padding: 2px;
+}
+
+.ACCEL {
+	background-color: #F0F0F0;
+	text-decoration: underline;
+} 
+
+.PROGRAMLISTING {
+	padding: 1ex;
+	background-color: #eee;
+	border: 1px solid #ccc;
+}
+
+@media screen {  /* hide from IE3 */:/
+	a[href]:hover { background: #ffa }
+}
+
+.NOTE, .SCREEN {
+	color: #222;
+	background: #fff;
+	border: 1px solid #ccc;
+	padding: 0.4em 0.4em;
+}
+
+.SCREEN {
+        background: #eee;
+}
+
+.TIP {
+	color: #004F00;
+	background: #fff;
+	border: 1px solid green;
+	padding: 0.4em 0.4em;
+}
+
+table.CDH {
+	padding:0px 0px 8px
+}
+
+table.CDH th {
+	background: #EDF2F9;
+	border-right: 0.5pt solid #adb6cd;
+	border-bottom: 0.5pt solid #adb6cd;
+        padding:2px 8px 1px;
+}
+
+table.CDH td {
+	font-size: 12px;
+}
+
+div.breadcrumbs {
+	border-top: 2px solid #0068BB;
+        background: #EDF2F9;
+	font-style: bold;
+	font-size: small;
+	padding: 0.6em 0.6em;
+}
+
+span.breadcrumb-node {
+	font-style: italic;
+}
+
+/* Match the the breadcrumbs at the top */
+div.footer-text {
+	border-bottom: 2px solid #0068BB;
+        background: #EDF2F9;
+        font-size: small;
+        padding: 0.2em 0.2em;
+	text-align: center;
+}
+
+.IMPORTANT, .caution {
+	font-style:italic;
+	border: 1px solid #a00;
+	border-left: 12px solid #c00;
+	padding: 0.1em 1em;
+}
+
+.WARNING {
+	color: #9F1313;
+	background: #fff;
+	border: 1px solid #e59595;
+	padding: 0.4em 0.4em;
+	width: 85%;
+}
+
+.EXAMPLE {
+	width: 100%;
+}
+
+.tip, .note, .caution {
+	margin:4px 0 4px;
+}
+
+img {
+	border: 0;
+}
diff --git a/flume-docs/src/docs/web/footer.xsl b/flume-docs/src/docs/web/footer.xsl
new file mode 100644
index 0000000..98582c0
--- /dev/null
+++ b/flume-docs/src/docs/web/footer.xsl
@@ -0,0 +1,47 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:exsl="http://exslt.org/common"
+                version="1.0"
+                exclude-result-prefixes="exsl">
+
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+
+<xsl:template name="user.footer.content">
+
+  <script type="text/javascript">
+     var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+     document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+  </script>
+  <script type="text/javascript">
+     try{
+        var pageTracker = _gat._getTracker("UA-2275969-4");
+        pageTracker._setDomainName(".cloudera.com");
+        pageTracker._trackPageview();
+     } catch(err) {}
+  </script>
+
+  <div class="footer-text">
+  <span align="center"><a href="index.html"><img src="images/home.png"
+      alt="Documentation Home" /></a></span>
+  <br/>
+  This document was built from Flume source available at
+  <a href="http://github.com/cloudera/flume">http://github.com/cloudera/flume</a>.
+  </div>
+
+</xsl:template>
+</xsl:stylesheet>
diff --git a/flume-docs/src/docs/web/header.xsl b/flume-docs/src/docs/web/header.xsl
new file mode 100644
index 0000000..7c74909
--- /dev/null
+++ b/flume-docs/src/docs/web/header.xsl
@@ -0,0 +1,41 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:exsl="http://exslt.org/common"
+                version="1.0"
+                exclude-result-prefixes="exsl">
+
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+
+<xsl:import href="breadcrumbs.xsl"/>
+
+<xsl:template name="user.head.content">
+</xsl:template>
+
+<xsl:template name="user.header.content">
+
+    <div style="clear:both; margin-bottom: 4px" />
+    <div align="center">
+      <a href="index.html"><img src="images/home.png"
+          alt="Documentation Home" /></a>
+    </div>
+    <span class="breadcrumbs">
+    <xsl:call-template name="breadcrumbs"/>
+    </span>
+
+</xsl:template>
+</xsl:stylesheet>
diff --git a/flume-docs/src/docs/web/html.xsl b/flume-docs/src/docs/web/html.xsl
new file mode 100644
index 0000000..2617dd4
--- /dev/null
+++ b/flume-docs/src/docs/web/html.xsl
@@ -0,0 +1,84 @@
+<?xml version='1.0'?>
+
+
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- $FreeBSD: doc/share/xsl/freebsd-html.xsl,v 1.1 2003/01/03 05:06:14 trhodes Exp $ -->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                version='1.0'
+                xmlns="http://www.w3.org/TR/xhtml1/transitional"
+                exclude-result-prefixes="#default">
+
+  <xsl:import href="header.xsl"/>
+  <xsl:import href="footer.xsl"/>
+
+  <xsl:template name="body.attributes" />
+
+  <xsl:param name="html.stylesheet" select="'docbook.css'"/>
+  <xsl:param name="use.id.as.filename" select="1"/>
+  <xsl:param name="generate.legalnotice.link" select="'1'"/>
+  <xsl:param name="generate.section.toc.level" select="100"></xsl:param>
+
+  <xsl:param name="toc.section.depth" select="10"/>
+  <xsl:param name="section.autolabel" select="1"/>
+  <xsl:param name="section.label.includes.component.label" select="1"/>
+  <xsl:param name="chunk.section.depth" select="100"></xsl:param>
+  <xsl:param name="chunk.first.sections" select="1"></xsl:param>
+  <xsl:param name="navig.showtitles" select="1"></xsl:param>
+
+  <xsl:param name="admon.graphics" select="1"></xsl:param>
+  <xsl:param name="admon.graphics.extension">.png</xsl:param>
+  <xsl:param name="admon.graphics.path">images/</xsl:param>
+
+  <xsl:param name="navig.graphics" select="1"></xsl:param>
+  <xsl:param name="navig.graphics.extension">.png</xsl:param>
+  <xsl:param name="navig.graphics.path">images/</xsl:param>
+
+  <xsl:param name="header.rule" select="0"></xsl:param>
+  <xsl:param name="footer.rule" select="0"></xsl:param>
+  <xsl:param name="suppress.header.navigation" select="1"></xsl:param>
+
+  <xsl:param name="generate.index" select="1"></xsl:param>
+
+  <xsl:param name="spacing.paras" select="0"></xsl:param>
+  <xsl:param name="html.cleanup" select="1"></xsl:param>
+
+  <xsl:param name="table.borders.with.css" select="1"></xsl:param>
+  <!-- xsl:param name="id.warnings" select="1"></xsl:param -->
+
+  <xsl:param name="generate.toc">
+appendix  toc,title
+article   toc,title,figure,equation
+book      toc,title,figure,example,equation
+chapter   toc,title
+part      toc,title
+preface   toc,title
+qandadiv  toc
+qandaset  toc
+reference toc,title
+sect1     toc
+sect2     toc
+sect3     toc
+sect4     toc
+sect5     toc
+section   toc
+set       toc,title
+   </xsl:param>
+
+</xsl:stylesheet>
diff --git a/flume-docs/src/docs/web/images/README b/flume-docs/src/docs/web/images/README
new file mode 100644
index 0000000..f12b2a7
--- /dev/null
+++ b/flume-docs/src/docs/web/images/README
@@ -0,0 +1,5 @@
+Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook
+icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency
+from the Jimmac icons to get round MS IE and FOP PNG incompatibilies.
+
+Stuart Rackham
diff --git a/flume-docs/src/docs/web/images/callouts/1.png b/flume-docs/src/docs/web/images/callouts/1.png
new file mode 100644
index 0000000000000000000000000000000000000000..7d473430b7bec514f7de12f5769fe7c5859e8c5d
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/10.png b/flume-docs/src/docs/web/images/callouts/10.png
new file mode 100644
index 0000000000000000000000000000000000000000..997bbc8246a316e040e0804174ba260e219d7d33
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/11.png b/flume-docs/src/docs/web/images/callouts/11.png
new file mode 100644
index 0000000000000000000000000000000000000000..ce47dac3f52ac49017749a3fea53db57d006993c
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/12.png b/flume-docs/src/docs/web/images/callouts/12.png
new file mode 100644
index 0000000000000000000000000000000000000000..31daf4e2f25b6712499ee32de9c2e3b050b691ca
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/13.png b/flume-docs/src/docs/web/images/callouts/13.png
new file mode 100644
index 0000000000000000000000000000000000000000..14021a89c2ed3d4881afea6e3a315bce4f95efce
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/14.png b/flume-docs/src/docs/web/images/callouts/14.png
new file mode 100644
index 0000000000000000000000000000000000000000..64014b75fe2e84d45ed861974c72462727979360
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/15.png b/flume-docs/src/docs/web/images/callouts/15.png
new file mode 100644
index 0000000000000000000000000000000000000000..0d65765fcf13dcfd87914744dec8bda115e4adf1
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/2.png b/flume-docs/src/docs/web/images/callouts/2.png
new file mode 100644
index 0000000000000000000000000000000000000000..5d09341b2f6d2ea2d1d5dad5d980f14b4b05dfd2
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/3.png b/flume-docs/src/docs/web/images/callouts/3.png
new file mode 100644
index 0000000000000000000000000000000000000000..ef7b70047158970cf4e09f1bab2954d39c2d596b
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/4.png b/flume-docs/src/docs/web/images/callouts/4.png
new file mode 100644
index 0000000000000000000000000000000000000000..adb8364eb5d21ecdd4086e16110b62ddcb42aa4a
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/5.png b/flume-docs/src/docs/web/images/callouts/5.png
new file mode 100644
index 0000000000000000000000000000000000000000..4d7eb460021e845981861d77614539314f553993
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/6.png b/flume-docs/src/docs/web/images/callouts/6.png
new file mode 100644
index 0000000000000000000000000000000000000000..0ba694af6c07d947d219b45a629bd32c60a0f5fe
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/7.png b/flume-docs/src/docs/web/images/callouts/7.png
new file mode 100644
index 0000000000000000000000000000000000000000..472e96f8ac36862c5645732f2fff19d06ee11a8e
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/8.png b/flume-docs/src/docs/web/images/callouts/8.png
new file mode 100644
index 0000000000000000000000000000000000000000..5e60973c213b37df93666c5a00724f34493974ae
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/callouts/9.png b/flume-docs/src/docs/web/images/callouts/9.png
new file mode 100644
index 0000000000000000000000000000000000000000..a0676d26cc2ff1de12c4ecdeefb44a0d71bc6bde
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/caution.png b/flume-docs/src/docs/web/images/caution.png
new file mode 100644
index 0000000000000000000000000000000000000000..cb9d5ea0df3fb7abe744d6ef885847c13b786d4d
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/example.png b/flume-docs/src/docs/web/images/example.png
new file mode 100644
index 0000000000000000000000000000000000000000..bba1c0010ddbb148a93e8a442047ca6a68b5ef1f
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/home.png b/flume-docs/src/docs/web/images/home.png
new file mode 100644
index 0000000000000000000000000000000000000000..37a5231bacc905886c9351a291e0d5eed68dd60e
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/important.png b/flume-docs/src/docs/web/images/important.png
new file mode 100644
index 0000000000000000000000000000000000000000..1096c232959e8cb5c2977b5eb44c1b67173cb422
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/next.png b/flume-docs/src/docs/web/images/next.png
new file mode 100644
index 0000000000000000000000000000000000000000..64e126bdda3f2e611199744494367ad0c1a1f9ca
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/note.png b/flume-docs/src/docs/web/images/note.png
new file mode 100644
index 0000000000000000000000000000000000000000..841820f7c4fb274fc45fae0a0a526a68eb31d85e
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/prev.png b/flume-docs/src/docs/web/images/prev.png
new file mode 100644
index 0000000000000000000000000000000000000000..3e8f12fe24c6e042bc7ef4badb15449f71529ace
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/tip.png b/flume-docs/src/docs/web/images/tip.png
new file mode 100644
index 0000000000000000000000000000000000000000..a3a029d89891bff666fbdc1a082ae4a7951b5abf
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/up.png b/flume-docs/src/docs/web/images/up.png
new file mode 100644
index 0000000000000000000000000000000000000000..2db1ce62fa575fd990108b15b0de18a64cae19ca
GIT binary patch
diff --git a/flume-docs/src/docs/web/images/warning.png b/flume-docs/src/docs/web/images/warning.png
new file mode 100644
index 0000000000000000000000000000000000000000..0b0c419df29b7e71ef14ecf5970b2abcd6d5ea01
GIT binary patch
diff --git a/flume-docs/src/docs/web/index.txt b/flume-docs/src/docs/web/index.txt
new file mode 100644
index 0000000..104a9ba
--- /dev/null
+++ b/flume-docs/src/docs/web/index.txt
@@ -0,0 +1,27 @@
+
+////
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+////
+
+Flume Documentation (v{revnumber})
+==================================
+
+The following pages are the documentation for Flume v{revnumber}:
+
+- link:UserGuide/index.html[Flume User Guide]
+- link:DeveloperGuide/index.html[Flume Developer Guide]
+- link:Cookbook/index.html[Flume Cookbook]
+- link:Windows/index.html[Flume Windows Guide]
diff --git a/pom.xml b/pom.xml
index 876c401..7b634fd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,6 +21,7 @@
     <module>flume-core</module>
     <module>flume-config-web</module>
     <module>flume-node-web</module>
+    <module>flume-docs</module>
     <module>flume-distribution</module>
     <module>flume-log4j-appender</module>
     <module>plugins/flume-plugin-helloworld</module>
-- 
1.7.1

