tutorial1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gensim Tutorial \n",
    "## Confirmation of LSA algorithm\n",
    "* Gensim official tutorial\n",
    "    * http://radimrehurek.com/gensim/tutorial.html]\n",
    "* Gensim tutorial (Japanese)\n",
    "    * Corupus and vector space http://hivecolor.com/id/58\n",
    "    * Topics and transformation http://hivecolor.com/id/59\n",
    "    * Similarity queries http://hivecolor.com/id/60\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['human', 'interface', 'computer'],\n",
      " ['survey', 'user', 'computer', 'system', 'response', 'time'],\n",
      " ['eps', 'user', 'interface', 'system'],\n",
      " ['system', 'human', 'system', 'eps'],\n",
      " ['user', 'response', 'time'],\n",
      " ['trees'],\n",
      " ['graph', 'trees'],\n",
      " ['graph', 'minors', 'trees'],\n",
      " ['graph', 'minors', 'survey']]\n"
     ]
    }
   ],
   "source": [
    "#import logging\n",
    "#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "from gensim import corpora, models, similarities\n",
    "\n",
    "documents = [\"Human machine interface for lab abc computer applications\",\n",
    "             \"A survey of user opinion of computer system response time\",\n",
    "             \"The EPS user interface management system\",\n",
    "             \"System and human system engineering testing of EPS\",\n",
    "             \"Relation of user perceived response time to error measurement\",\n",
    "             \"The generation of random binary unordered trees\",\n",
    "             \"The intersection graph of paths in trees\",\n",
    "             \"Graph minors IV Widths of trees and well quasi ordering\",\n",
    "             \"Graph minors A survey\"]\n",
    "# remove common words and tokenize\n",
    "stoplist = set('for a of the and to in'.split())\n",
    "texts = [[word for word in document.lower().split() if word not in stoplist]\n",
    "          for document in documents]\n",
    "#pprint(texts)\n",
    "#pprint(\"-------------\")\n",
    "\n",
    "# remove words that appear only once\n",
    "from collections import defaultdict\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "    for token in text: \n",
    "         frequency[token] += 1\n",
    "\n",
    "texts = [[token for token in text if frequency[token] > 1]\n",
    "          for text in texts]\n",
    "\n",
    "from pprint import pprint  # pretty-printer\n",
    "pprint(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-01-08 00:58:53,454 : INFO : adding document #0 to Dictionary(0 unique tokens: [])\n",
      "2017-01-08 00:58:53,470 : INFO : built Dictionary(12 unique tokens: ['survey', 'interface', 'computer', 'graph', 'user']...) from 9 documents (total 29 corpus positions)\n",
      "2017-01-08 00:58:53,470 : INFO : saving dictionary mapping to .\\deerwester_text.dict\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(12 unique tokens: ['survey', 'interface', 'computer', 'graph', 'user']...)\n",
      "{'survey': 3, 'interface': 0, 'computer': 1, 'graph': 10, 'user': 7, 'time': 5, 'response': 6, 'system': 4, 'minors': 11, 'human': 2, 'trees': 9, 'eps': 8}\n"
     ]
    }
   ],
   "source": [
    "dictionary = corpora.Dictionary(texts)\n",
    "#dictionary.save('/deerwester.dict')  # store the dictionary, for future reference\n",
    "# バイナリではなくテキストとして保存する場合\n",
    "dictionary.save_as_text('.\\deerwester_text.dict')\n",
    "print(dictionary)\n",
    "\n",
    "#各トークンのIDを表示\n",
    "print (dictionary.token2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-01-08 00:58:53,548 : INFO : storing corpus in Matrix Market format to .\\deerwester.mm\n",
      "2017-01-08 00:58:53,564 : INFO : saving sparse matrix to .\\deerwester.mm\n",
      "2017-01-08 00:58:53,564 : INFO : PROGRESS: saving document #0\n",
      "2017-01-08 00:58:53,564 : INFO : saved 9x12 matrix, density=25.926% (28/108)\n",
      "2017-01-08 00:58:53,579 : INFO : saving MmCorpus index to .\\deerwester.mm.index\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(1, 1), (2, 1)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-01-08 00:58:53,595 : INFO : loaded corpus index from .\\deerwester.mm.index\n",
      "2017-01-08 00:58:53,595 : INFO : initializing corpus reader from .\\deerwester.mm\n",
      "2017-01-08 00:58:53,595 : INFO : accepted corpus with 9 documents, 12 features, 28 non-zero entries\n",
      "2017-01-08 00:58:53,611 : INFO : collecting document frequencies\n",
      "2017-01-08 00:58:53,611 : INFO : PROGRESS: processing document #0\n",
      "2017-01-08 00:58:53,611 : INFO : calculating IDF weights for 9 documents and 11 features (28 matrix non-zeros)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MmCorpus(9 documents, 12 features, 28 non-zero entries)\n",
      "-------------------tutorial2\n",
      "[(1, 1), (2, 1)]\n",
      "[(1, 0.7071067811865476), (2, 0.7071067811865476)]\n",
      "---tfidf---------------\n",
      "[(0, 0.5773502691896257), (1, 0.5773502691896257), (2, 0.5773502691896257)]\n",
      "[(1, 0.44424552527467476), (3, 0.44424552527467476), (4, 0.3244870206138555), (5, 0.44424552527467476), (6, 0.44424552527467476), (7, 0.3244870206138555)]\n",
      "[(0, 0.5710059809418182), (4, 0.4170757362022777), (7, 0.4170757362022777), (8, 0.5710059809418182)]\n",
      "[(2, 0.49182558987264147), (4, 0.7184811607083769), (8, 0.49182558987264147)]\n",
      "[(5, 0.6282580468670046), (6, 0.6282580468670046), (7, 0.45889394536615247)]\n",
      "[(9, 1.0)]\n",
      "[(9, 0.7071067811865475), (10, 0.7071067811865475)]\n",
      "[(9, 0.5080429008916749), (10, 0.5080429008916749), (11, 0.695546419520037)]\n",
      "[(3, 0.6282580468670046), (10, 0.45889394536615247), (11, 0.6282580468670046)]\n"
     ]
    }
   ],
   "source": [
    "# doc2bow()はdistinctした結果から単語の出現回数を数えます。\n",
    "#[(0, 1), (1, 1)]という出力結果から、computerの辞書内のIDは0で今回の出現回数1回、\n",
    "#humanの辞書内のIDは1で今回の出現回数は1回、interctionは辞書に入っていない(なので無視された)、ということが分かります。\n",
    "new_doc = \"Human computer interaction\"\n",
    "new_vec = dictionary.doc2bow(new_doc.lower().split())\n",
    "print (new_vec) # the word \"interaction\" does not appear in the dictionary and is ignored\n",
    "\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "corpora.MmCorpus.serialize('.\\deerwester.mm', corpus) # store to disk, for later use\n",
    "corpus = corpora.MmCorpus('.\\deerwester.mm')\n",
    "\n",
    "# print list(corpus)\n",
    "print (corpus)\n",
    "print (\"-------------------tutorial2\")\n",
    "tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model\n",
    "print (new_vec)\n",
    "print (tfidf[new_vec]) # step 2 -- use the model to transform vectors\n",
    "print (\"---tfidf---------------\")\n",
    "corpus_tfidf = tfidf[corpus]\n",
    "for doc in corpus_tfidf:\n",
    "    print (doc)\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-01-08 00:58:53,673 : INFO : using serial LSI version on this node\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---lsi---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-01-08 00:58:53,673 : INFO : updating model with new documents\n",
      "2017-01-08 00:58:53,689 : INFO : preparing a new chunk of documents\n",
      "2017-01-08 00:58:53,689 : INFO : using 100 extra samples and 2 power iterations\n",
      "2017-01-08 00:58:53,689 : INFO : 1st phase: constructing (12, 102) action matrix\n",
      "2017-01-08 00:58:53,689 : INFO : orthonormalizing (12, 102) action matrix\n",
      "2017-01-08 00:58:53,704 : INFO : 2nd phase: running dense svd on (12, 9) matrix\n",
      "2017-01-08 00:58:53,704 : INFO : computing the final decomposition\n",
      "2017-01-08 00:58:53,704 : INFO : keeping 2 factors (discarding 47.565% of energy spectrum)\n",
      "2017-01-08 00:58:53,704 : INFO : processed documents up to #9\n",
      "2017-01-08 00:58:53,720 : INFO : topic #0(1.594): 0.703*\"trees\" + 0.538*\"graph\" + 0.402*\"minors\" + 0.187*\"survey\" + 0.061*\"system\" + 0.060*\"response\" + 0.060*\"time\" + 0.058*\"user\" + 0.049*\"computer\" + 0.035*\"interface\"\n",
      "2017-01-08 00:58:53,720 : INFO : topic #1(1.476): -0.460*\"system\" + -0.373*\"user\" + -0.332*\"eps\" + -0.328*\"interface\" + -0.320*\"response\" + -0.320*\"time\" + -0.293*\"computer\" + -0.280*\"human\" + -0.171*\"survey\" + 0.161*\"trees\"\n",
      "2017-01-08 00:58:53,720 : INFO : topic #0(1.594): 0.703*\"trees\" + 0.538*\"graph\" + 0.402*\"minors\" + 0.187*\"survey\" + 0.061*\"system\" + 0.060*\"response\" + 0.060*\"time\" + 0.058*\"user\" + 0.049*\"computer\" + 0.035*\"interface\"\n",
      "2017-01-08 00:58:53,736 : INFO : topic #1(1.476): -0.460*\"system\" + -0.373*\"user\" + -0.332*\"eps\" + -0.328*\"interface\" + -0.320*\"response\" + -0.320*\"time\" + -0.293*\"computer\" + -0.280*\"human\" + -0.171*\"survey\" + 0.161*\"trees\"\n",
      "2017-01-08 00:58:53,736 : INFO : saving Projection object under .\\model.lsi.projection, separately None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.066007833960905024), (1, -0.5200703306361848)]\n",
      "[(0, 0.19667592859142788), (1, -0.76095631677000386)]\n",
      "[(0, 0.089926399724467129), (1, -0.72418606267525054)]\n",
      "[(0, 0.075858476521784346), (1, -0.63205515860034267)]\n",
      "[(0, 0.10150299184980362), (1, -0.57373084830029486)]\n",
      "[(0, 0.70321089393783065), (1, 0.16115180214026062)]\n",
      "[(0, 0.8774787673119826), (1, 0.16758906864659731)]\n",
      "[(0, 0.90986246868185727), (1, 0.14086553628719348)]\n",
      "[(0, 0.61658253505692828), (1, -0.053929075663891483)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-01-08 00:58:53,767 : INFO : saved .\\model.lsi.projection\n",
      "2017-01-08 00:58:53,767 : INFO : saving LsiModel object under .\\model.lsi, separately None\n",
      "2017-01-08 00:58:53,767 : INFO : not storing attribute projection\n",
      "2017-01-08 00:58:53,783 : INFO : not storing attribute dispatcher\n",
      "2017-01-08 00:58:53,783 : INFO : saved .\\model.lsi\n",
      "2017-01-08 00:58:53,783 : INFO : loading LsiModel object from .\\model.lsi\n",
      "2017-01-08 00:58:53,798 : INFO : loading id2word recursively from .\\model.lsi.id2word.* with mmap=None\n",
      "2017-01-08 00:58:53,798 : INFO : setting ignored attribute projection to None\n",
      "2017-01-08 00:58:53,798 : INFO : setting ignored attribute dispatcher to None\n",
      "2017-01-08 00:58:53,798 : INFO : loaded .\\model.lsi\n",
      "2017-01-08 00:58:53,798 : INFO : loading LsiModel object from .\\model.lsi.projection\n",
      "2017-01-08 00:58:53,814 : INFO : loaded .\\model.lsi.projection\n"
     ]
    }
   ],
   "source": [
    "  \n",
    "print (\"---lsi---------------\")   \n",
    "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2) # initialize an LSI transformation\n",
    "corpus_lsi = lsi[corpus_tfidf] # create a double wrapper over the original corpus: bow->tfidf->fold-in-lsi\n",
    "lsi.print_topics(2)\n",
    "\n",
    "for doc in corpus_lsi: # both bow->tfidf and tfidf->lsi transformations are actually executed here, on the fly\n",
    "    print (doc)\n",
    "    \n",
    "lsi.save('.\\model.lsi') # same for tfidf, lda, ...\n",
    "lsi = models.LsiModel.load('.\\model.lsi')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}