updated manuals

arangodb · moonglum · Oct 8, 2013 · Oct 8, 2013 · Oct 8, 2013 · Oct 8, 2013
commit 956bd530b79514a1baa60ff672dd5e00849ded82
diff --git a/Documentation/ImporterManual/BasicsTOC.md b/Documentation/ImporterManual/BasicsTOC.md
diff --git a/Documentation/ImporterManual/ImporterManual.md b/Documentation/ImporterManual/ImporterManual.md
diff --git a/Documentation/Makefile.files b/Documentation/Makefile.files
@@ -25,6 +25,7 @@ WIKI = \
 	DbaManualBasics \
 	DbaManualDatafileDebugger \
 	DbaManualEmergencyConsole \
+	DumpManual \
 	ExtendingAql \
 	FirstStepsArangoDB \
 	Glossary \
@@ -55,7 +56,6 @@ WIKI = \
 	HttpTraversals \
 	HttpUser \
 	ImpManual \
-	ImpManualBasics \
 	ImplementorManual \
 	ImplementorManualArangoErrors \
 	IndexBitArray \
@@ -81,7 +81,9 @@ WIKI = \
 	RefManualReplication \
 	RestDocument \
 	RestEdge \
+	RestoreManual \
 	SimpleQueries \
+	ToolsManual \
 	Transactions \
 	Upgrading \
 	UpgradingGeneral \
@@ -108,6 +110,7 @@ DOXYGEN_TOC = \
 	Documentation/InstallationManual/InstallationManual.md \
 	Documentation/Manual/Home.md \
 	Documentation/RefManual/RefManual.md \
+	Documentation/ToolsManual/ToolsManual.md \
 	Documentation/UserManual/UserManual.md \
 	Documentation/Manual/Upgrading.md
 
@@ -289,8 +292,8 @@ latex: Doxygen/.setup-directories Doxygen/arango-latex.doxy $(DOXYGEN)
 	python @top_srcdir@/Documentation/Scripts/tex2tex.py Doxygen/latex/DbaManual.tex > Doxygen/latex/DbaManual.inc.tex
 	cd Doxygen/latex && pdflatex -interaction batchmode dba-manual.tex || true
 
-	python @top_srcdir@/Documentation/Scripts/tex2tex.py Doxygen/latex/ImpManual.tex > Doxygen/latex/ImpManual.inc.tex
-	cd Doxygen/latex && pdflatex -interaction batchmode imp-manual.tex || true
+	python @top_srcdir@/Documentation/Scripts/tex2tex.py Doxygen/latex/ToolsManual.tex > Doxygen/latex/ToolsManual.inc.tex
+	cd Doxygen/latex && pdflatex -interaction batchmode tools-manual.tex || true
 
 ################################################################################
 ### @brief man pages
@@ -391,7 +394,7 @@ publish-pdf: $(OUTPUT_FOLDER)
 	cp Doxygen/latex/install-manual.pdf $(OUTPUT_FOLDER)
 	cp Doxygen/latex/user-manual.pdf $(OUTPUT_FOLDER)
 	cp Doxygen/latex/ref-manual.pdf $(OUTPUT_FOLDER)
-	cp Doxygen/latex/imp-manual.pdf $(OUTPUT_FOLDER)
+	cp Doxygen/latex/tools-manual.pdf $(OUTPUT_FOLDER)
 	cp Documentation/Manual/shell_reference_card.pdf $(OUTPUT_FOLDER)
 
 ## -----------------------------------------------------------------------------

diff --git a/Documentation/Manual/Home.md b/Documentation/Manual/Home.md
@@ -37,7 +37,7 @@ ArangoDB's User Manuals {#ArangoDBUserManual}
 
 - @BOOK_REF{DbaManual} @S_EXTREF{dba-manual.pdf,pdf}
 
-- @BOOK_REF{ImpManual} @S_EXTREF{imp-manual.pdf,pdf}
+- @BOOK_REF{ToolsManual} @S_EXTREF{tools-manual.pdf,pdf}
 
 - also see the @ref Glossary
 

diff --git a/Documentation/ToolsManual/DumpManual.md b/Documentation/ToolsManual/DumpManual.md
@@ -0,0 +1,70 @@
+Dumping Data from an ArangoDB database {#DumpManual}
+====================================================
+
+@NAVIGATE_DumpManual
+@EMBEDTOC{DumpManualTOC}
+
+To dump data from an ArangoDB server instance, you will need to invoke _arangodump_.
+It can be invoked by executing the following command:
+
+    unix> arangodump --output-directory "dump"
+
+This will connect to an ArangoDB server and dump all non-system collections from
+the default database (`_system`) into an output directory named `dump`.
+Invoking _arangodump_ will fail if the output directory already exists. This is
+an intentional security measure to prevent you from accidently overwriting already
+dumped data. If you are positive that you want to overwrite data in the output 
+directory, you can use the parameter `--overwrite true` to confirm this:
+
+    unix> arangodump --output-directory "dump" --overwrite true
+
+_arangodump_ will by default connect to the `_system` database using the default
+endpoint. If you want to connect to a different database or a different endpoint, 
+or use authentication, you can use the following command-line options:
+
+- `--server.database <string>`: name of the database to connect to
+- `--server.endpoint <string>`: endpoint to connect to
+- `--server.username <string>`: username
+- `--server.password <string>`: password to use (omit this and you'll be prompted for the
+  password)
+- `--server.disable-authentication <bool>`: whether or not to use authentication
+
+Here's an example of dumping data from a non-standard endpoint, using a dedicated
+database name:
+
+    unix> arangodump --server.endpoint tcp://192.168.173.13:8531 --server.username backup --server.database mydb --output-directory "dump"
+
+When finished, _arangodump_ will print out a summary line with some aggregate 
+statistics about what it did, e.g.:
+
+    Processed 43 collection(s), wrote 408173500 byte(s) into datafiles, sent 88 batch(es)
+
+By default, _arangodump_ will dump both structural information and documents from all
+non-system collections. To adjust this, there are the following command-line 
+arguments:
+
+- `--dump-data <bool>`: set to `true` to include documents in the dump. Set to `false` 
+  to exclude documents. The default value is `true`.
+- `--include-system-collections <bool>`: whether or not to include system collections
+  in the dump. The default value is `false`.
+
+For example, to only dump structural information of all collections (including system
+collections), use:
+
+    unix> arangodump --dump-data false --include-system-collections true --output-directory "dump"
+
+To restrict the dump to just specific collections, there is is the `--collection` option.
+It can be specified multiple times if required:
+
+    unix> arangodump --collection myusers --collection myvalues --output-directory "dump"
+
+Structural information for a collection will be saved in files with name pattern 
+`<collection-name>.structure.json`. Each structure file will contains a JSON object 
+with these attributes:
+- `parameters`: contains the collection properties
+- `indexes`: contains the collection indexes
+
+Document data for a collection will be saved in files with name pattern 
+`<collection-name>.data.json`. Each line in a data file is a document insertion/update or
+deletion marker, alongside with some meta data.
+
diff --git a/Documentation/ToolsManual/DumpManualTOC.md b/Documentation/ToolsManual/DumpManualTOC.md
@@ -0,0 +1,4 @@
+TOC {#DumpManualTOC}
+====================
+
+- @ref DumpManual
diff --git a/Documentation/ImporterManual/Basics.md → Documentation/ToolsManual/ImpManual.md b/Documentation/ImporterManual/Basics.md → Documentation/ToolsManual/ImpManual.md
@@ -1,9 +1,8 @@
-ArangoDB Importer {#ImpManualBasics}
-====================================
+Importing Data into an ArangoDB Database {#ImpManual}
+=====================================================
 
 @NAVIGATE_ImpManual
-
-@EMBEDTOC{ImpManualBasicsTOC}
+@EMBEDTOC{ImpManualTOC}
 
 This manual describes the ArangoDB importer _arangoimp_, which can be used for
 bulk imports.
@@ -21,8 +20,8 @@ valid references.
 Let's assume for the following examples you want to import user records into an 
 existing collection named "users" on the server.
 
-Importing JSON-encoded Data {#ImpManualBasicsJson}
-==================================================
+Importing JSON-encoded Data {#ImpManualJson}
+============================================
 
 Let's further assume the import at hand is encoded in JSON. We'll be using these
 example user records to import:
@@ -66,8 +65,8 @@ Please note that by default, _arangoimp_ will import data into the specified
 collection in the default database (`_system`). To specify a different database, 
 use the `--server.database` option when invoking _arangoimp_. 
 
-Importing CSV Data {#ImpManualBasicsCsv}
-========================================
+Importing CSV Data {#ImpManualCsv}
+==================================
 
 _arangoimp_ also offers the possibility to import data from CSV files. This
 comes handy when the data at hand is in CSV format already and you don't want to
@@ -98,8 +97,8 @@ Note that the quote and separator characters can be adjusted via the
 `--quote` and `--separator` arguments when invoking _arangoimp_.  The importer
 supports Windows (CRLF) and Unix (LF) line breaks.
 
-Importing TSV Data {#ImpManualBasicsTsv}
-========================================
+Importing TSV Data {#ImpManualTsv}
+==================================
 
 You may also import tab-separated values (TSV) from a file. This format is very
 simple: every line in the file represents a data record. There is no quoting or

diff --git a/Documentation/ToolsManual/ImpManualTOC.md b/Documentation/ToolsManual/ImpManualTOC.md
@@ -0,0 +1,7 @@
+TOC {#ImpManualTOC}
+===================
+
+- @ref ImpManual
+  - @ref ImpManualJson
+  - @ref ImpManualCsv
+  - @ref ImpManualTsv
diff --git a/Documentation/ToolsManual/RestoreManual.md b/Documentation/ToolsManual/RestoreManual.md
@@ -0,0 +1,130 @@
+Reloading Data into an ArangoDB database {#RestoreManual}
+=========================================================
+
+@NAVIGATE_RestoreManual
+@EMBEDTOC{RestoreManualTOC}
+
+To reload data from a dump previously created with @ref DumpManual "arangodump",
+ArangoDB provides the _arangorestore_ tool.
+
+Invoking arangorestore {#RestoreManualInvoking}
+===============================================
+
+_arangorestore_ can be invoked from the command-line as follows:
+
+    unix> arangorestore --input-directory "dump"
+
+This will connect to an ArangoDB server and reload structural information and 
+documents found in the input directory `dump`. Please note that the input directory
+must have been created by running `arangodump` before.
+
+_arangorestore_ will by default connect to the `_system` database using the default
+endpoint. If you want to connect to a different database or a different endpoint, 
+or use authentication, you can use the following command-line options:
+
+- `--server.database <string>`: name of the database to connect to
+- `--server.endpoint <string>`: endpoint to connect to
+- `--server.username <string>`: username
+- `--server.password <string>`: password to use (omit this and you'll be prompted for the
+  password)
+- `--server.disable-authentication <bool>`: whether or not to use authentication
+
+Here's an example of reloading data to a non-standard endpoint, using a dedicated
+database name:
+
+    unix> arangorestore --server.endpoint tcp://192.168.173.13:8531 --server.username backup --server.database mydb --input-directory "dump"
+
+_arangorestore_ will print out its progress while running, and will end with a line
+showing some aggregate statistics:
+
+    Processed 2 collection(s), read 2256 byte(s) from datafiles, sent 2 batch(es)
+
+
+By default, _arangorestore_ will re-create all non-system collections found in the input 
+directory and load data into them. If the target database already contains collections 
+which are also present in the input directory, the existing collections in the database 
+will be dropped and re-created with the data found in the input directory.
+
+The following parameters are available to adjust this behavior:
+
+- `--create-collection <bool>`: set to `true` to create collections in the target
+  database. If the target database already contains a collection with the same name,
+  it will be dropped first and then re-created with the properties found in the input
+  directory. Set to `false` to keep existing collections in the target database. If 
+  set to `false` and _arangorestore_ encounters a collection that is present in both 
+  the target database and the input directory, it will abort. The default value is `true`.
+- `--import-data <bool>`: set to `true` to load document data into the collections in
+  the target database. Set to `false` to not load any document data. The default value 
+  is `true`.
+- `--include-system-collections <bool>`: whether or not to include system collections
+  when re-creating collections or reloading data. The default value is `false`.
+
+For example, to (re-)create all non-system collections and load document data into them, use:
+
+    unix> arangorestore --create-collection true --import-data true --input-directory "dump"
+
+This will drop potentially existing collections in the target database that are also present
+in the input directory.
+
+To include system collections too, use `--include-system-collections true`:
+
+    unix> arangorestore --create-collection true --import-data true --include-system-collections true --input-directory "dump"
+
+To (re-)create all non-system collections without loading document data, use:
+
+    unix> arangorestore --create-collection true --import-data false --input-directory "dump"
+
+This will also drop existing collections in the target database that are also present in the
+input directory.
+
+To just load document data into all non-system collections, use:
+
+    unix> arangorestore --create-collection false --import-data true --input-directory "dump"
+
+To restrict reloading to just specific collections, there is is the `--collection` option.
+It can be specified multiple times if required:
+
+    unix> arangorestore --collection myusers --collection myvalues --input-directory "dump"
+
+Collections will be processed by in alphabetical order by _arangorestore_, with all document
+collections being processed before all edge collections. This is to ensure that reloading
+data into edge collections will have the document collections linked in edges (`_from` and
+`_to` attributes) loaded.
+
+Restoring Revision Ids and Collection Ids {#RestoreManualIds}
+=============================================================
+
+_arangorestore_ will reload document and edges data with the exact same `_key`, `_from` and 
+`_to` values found in the input directory. However, when loading document data, it will assign
+its own values for the `_rev` attribute of the reloaded documents. Though this difference is 
+intentional (normally, every server should create its own `_rev` values) there might be 
+situations when it is required to re-use the exact same `_rev` values for the reloaded data.
+This can be achieved by setting the `--recycle-ids` parameter to `true`:
+
+    unix> arangorestore --collection myusers --collection myvalues --recycle-ids true --input-directory "dump"
+
+Note that setting `--recycle-ids` to `true` will also cause collections to be (re-)created in
+the target database with the exact same collection id as in the input directory. Any potentially
+existing collection in the target database with the same collection id will then be dropped.
+
+Setting `--recycle-ids` to `false` or omitting it will only use the collection name from the
+input directory and allow the target database to create the collection with a different id
+(though with the same name) than in the input directory.
+
+Reloading Data into a different Collection {#RestoreManualDifferent}
+====================================================================
+
+With some creativity you can use _arangodump_ and _arangorestore_ to transfer data from one
+collection into another (either on the same server or not). For example, to copy data from
+a collection `myvalues` in database `mydb` into a collection `mycopyvalues` in database `mycopy`,
+you can start with the following command:
+
+    unix> arangodump --collection myvalues --server.database mydb --output-directory "dump"
+
+This will create two files, `myvalues.structure.json` and `myvalues.data.json`, in the output 
+directory. To load data from the datafile into an existing collection `mycopyvalues` in database 
+`mycopy`, rename the files to `mycopyvalues.structure.json` and `mycopyvalues.data.json`.
+After that, run the following command:
+
+    unix> arangorestore --collection mycopyvalues --server.database mycopy --input-directory "dump"
+
diff --git a/Documentation/ToolsManual/RestoreManualTOC.md b/Documentation/ToolsManual/RestoreManualTOC.md
@@ -0,0 +1,7 @@
+TOC {#RestoreManualTOC}
+=======================
+
+- @ref RestoreManual
+  - @ref RestoreManualInvoking
+  - @ref RestoreManualIds
+  - @ref RestoreManualDifferent
diff --git a/Documentation/ToolsManual/ToolsManual.md b/Documentation/ToolsManual/ToolsManual.md
@@ -0,0 +1,35 @@
+ArangoDB Client Tools (@VERSION) {#ToolsManual}
+===============================================
+
+@NAVIGATE_ToolsManual
+
+This manual describes the client tools shipped with ArangoDB.
+
+_arangosh_ is an interactive, JavaScript-enabled shell that can be used to administer
+an ArangoDB server and run ad-hoc queries:
+
+@CHAPTER_REF{UserManualArangosh}
+
+_arangoimp_ is a tool to bulk insert data from JSON, CSV or tab-separated files (TSV)
+files into an ArangoDB database:
+
+@CHAPTER_REF{ImpManual}
+
+ArangoDB since version 1.4 comes with two additional tools to support dumping data
+from an ArangoDB database and reloading them. 
+
+_arangodump_ is a tool to create logical backups of one or many collections of an 
+ArangoDB database:
+
+@CHAPTER_REF{DumpManual}
+
+_arangorestore_ is a tool to reload data dumped with _arangodump_ into an
+ArangoDB database. Data can be loaded into the same or a different database:
+
+@CHAPTER_REF{RestoreManual}
+
+Both _arangodump_ and _arangorestore_ are client tools that need to connect to a 
+running ArangoDB server instance. _arangodump_ will write all its output into a 
+directory that needs to be specified when invoking it. _arangorestore_ will read
+files from a directory that was formerly created by invoking _arangodump_.
+