javascript - Mongoose won't save huge text - Stack Overflow

admin管理员组

文章数量:1410674

I'm trying to save a very big text into my mongodb database but it crash. I'm trying to build a snippet manager and this is my error code:

{ [MongoError: Btree::insert: key too large to index, failing beverages.recipes.$code_1 3271 { : "var gulp = require('gulp');
var source = require('vinyl-source-stream');
var browserify = require('browserify');
var browserSync = require('browser-sy..." }]
  name: 'MongoError',
  message: 'Btree::insert: key too large to index, failing beverages.recipes.$code_1 3271 { : "var gulp = require(\'gulp\');\nvar source = require(\'vinyl-source-stream\');\nvar browserify = require(\'browserify\');\nvar browserSync = require(\'browser-sy..." }',
  driver: true,
  code: 17280,
  index: 0,
  errmsg: 'Btree::insert: key too large to index, failing beverages.recipes.$code_1 3271 { : "var gulp = require(\'gulp\');\nvar source = require(\'vinyl-source-stream\');\nvar browserify = require(\'browserify\');\nvar browserSync = require(\'browser-sy..." }',
  getOperation: [Function],
  toJSON: [Function],
  toString: [Function] }

This is my Mongoose Model.

import mongoose from 'mongoose';

const Schema = mongoose.Schema;

let recipeSchema = new Schema({
  title: { type: String, required: true  },
  author: { type: String},
  tags: { type: String, required: true },
  code: { type: String, required: true, unique: true, index: true },
  created_at: Date,
  updated_at: Date
});

recipeSchema.pre('save', function(next) {

  var recipe = this;
  // get the current date
  var currentDate = new Date();

  // change the updated_at field to current date
  recipe.updated_at = currentDate;

  // if created_at doesn't exist, add to that field
  if (!recipe.created_at){
    recipe.created_at = currentDate;
  }

  next();
});

let Recipe = mongoose.model('Recipe', recipeSchema);

export default Recipe;

If found this on mongodb, but i don't know how to user it with mongoose. Is it the right thing to do?

This is the document i try to save

{ title: 'test',
  tags: 'test',
  code: 'var gulp = require(\'gulp\');\nvar source = require(\'vinyl-source-stream\');\nvar browserify = require(\'browserify\');\nvar browserSync = require(\'browser-sync\');\nvar nodemon = require(\'gulp-nodemon\');\nvar babelify = require(\'babelify\');\nvar eslint = require(\'gulp-eslint\');\nvar buffer = require(\'vinyl-buffer\');\nvar uglify = require(\'gulp-uglify\');\nvar sourcemaps = require(\'gulp-sourcemaps\');\nvar sass = require(\'gulp-ruby-sass\');\nvar config = require(\'./config\');\nvar imageop = require(\'gulp-image-optimization\');\nvar concat = require(\'gulp-concat\');\n\ngulp.task(\'images\', function(cb) {\n  gulp.src(\'./src/assets/images/**/*\')\n    .pipe( imageop({\n      optimizationLevel: 5,\n      progressive: true,\n      interlaced: true\n    }))\n    .pipe(gulp.dest(\'./public/images/\'));\n});\n\n\ngulp.task(\'browser-sync\', [\'nodemon\'], function() {\n  browserSync({\n    files: [\'public/**/*.*\'],\n    proxy: "localhost:" + config.APP_PORT,  // local node app address\n    port: 5000,  // use *different* port than above\n    notify: true,\n    browser: "google chrome",\n  });\n});\n\ngulp.task(\'nodemon\', function(cb) {\n  var called = false;\n  return nodemon({\n    script: \'./index.js\',\n    ignore: [\n      \'gulpfile.js\',\n      \'node_modules/\',\n      \'public/,\'\n    ]\n  })\n    .on(\'start\', function () {\n      if (!called) {\n        called = true;\n        cb();\n      }\n    })\n    .on(\'restart\', function () {\n      setTimeout(function () {\n        browserSync.reload({ stream: false });\n      }, 200);\n    });\n});\n\n\ngulp.task(\'browserify\', function () {\n  return browserify({entries: \'./src/client/app.js\', extensions: [\'.js\'], debug: true})\n    .transform(babelify)\n    .bundle()\n    .pipe(source(\'app.min.js\'))\n    .pipe(buffer())\n    .pipe(sourcemaps.init({loadMaps: true}))\n    // Add transformation tasks to the pipeline here.\n    .pipe(uglify())\n    .pipe(sourcemaps.write(\'./\'))\n    .pipe(gulp.dest(\'./public/js\'));\n});\n\ngulp.task(\'lint\', function (cb) {\n  return gulp.src([\'./src/**/*.js\'])\n    .pipe(eslint())\n    .pipe(eslint.format())\n    .pipe(eslint.failOnError());\n});\n\ngulp.task(\'sass\', function () {\n  return sass(\'./src/assets/sass/style.sass\', { sourcemap: true,  style: \'pressed\' })\n    .on(\'error\', sass.logError)\n    .pipe(sourcemaps.write(\'./\', {\n      includeContent: false,\n      sourceRoot: \'source\'\n    }))\n    .pipe(gulp.dest(\'./public/css/\'));\n});\n\ngulp.task(\'scripts\', function(){\n  return gulp.src([\'./src/assets/js/bootstrap.min.js\',\'./src/assets/js/checkbox.js\',\'./src/assets/js/radio.js\',\'./src/assets/js/bootstrap-switch.js\',\'./src/assets/js/toolbar.js\',\'./src/assets/js/application.js\'])\n    .pipe(sourcemaps.init())\n    .pipe(concat(\'assets.min.js\'))\n    .pipe(gulp.dest(\'./public/js/\'))\n    .pipe(uglify())\n    .pipe(sourcemaps.write(\'./\'))\n    .pipe(gulp.dest(\'./public/js/\'));\n});\n\ngulp.task(\'fonts\', function() {\n  return gulp.src(\'./src/assets/fonts/**/*\').pipe(gulp.dest(\'./public/fonts/\'));\n});\n\ngulp.task(\'watch\', function() {\n  gulp.watch([\'./src/**/*.js\'], [\'lint\', \'browserify\', \'scripts\', browserSync.reload]);\n  gulp.watch([\'./src/assets/sass/**/*.sass\'], [\'sass\', browserSync.reload]);\n});\n\ngulp.task(\'serve\', [\'browser-sync\', \'watch\']);\n\ngulp.task(\'default\', [\'browserify\', \'nodemon\', \'sass\', \'scripts\', \'fonts\', \'images\', \'watch\']);' }

Thank you very much for your help!

I'm trying to save a very big text into my mongodb database but it crash. I'm trying to build a snippet manager and this is my error code:

{ [MongoError: Btree::insert: key too large to index, failing beverages.recipes.$code_1 3271 { : "var gulp = require('gulp');
var source = require('vinyl-source-stream');
var browserify = require('browserify');
var browserSync = require('browser-sy..." }]
  name: 'MongoError',
  message: 'Btree::insert: key too large to index, failing beverages.recipes.$code_1 3271 { : "var gulp = require(\'gulp\');\nvar source = require(\'vinyl-source-stream\');\nvar browserify = require(\'browserify\');\nvar browserSync = require(\'browser-sy..." }',
  driver: true,
  code: 17280,
  index: 0,
  errmsg: 'Btree::insert: key too large to index, failing beverages.recipes.$code_1 3271 { : "var gulp = require(\'gulp\');\nvar source = require(\'vinyl-source-stream\');\nvar browserify = require(\'browserify\');\nvar browserSync = require(\'browser-sy..." }',
  getOperation: [Function],
  toJSON: [Function],
  toString: [Function] }

This is my Mongoose Model.

import mongoose from 'mongoose';

const Schema = mongoose.Schema;

let recipeSchema = new Schema({
  title: { type: String, required: true  },
  author: { type: String},
  tags: { type: String, required: true },
  code: { type: String, required: true, unique: true, index: true },
  created_at: Date,
  updated_at: Date
});

recipeSchema.pre('save', function(next) {

  var recipe = this;
  // get the current date
  var currentDate = new Date();

  // change the updated_at field to current date
  recipe.updated_at = currentDate;

  // if created_at doesn't exist, add to that field
  if (!recipe.created_at){
    recipe.created_at = currentDate;
  }

  next();
});

let Recipe = mongoose.model('Recipe', recipeSchema);

export default Recipe;

If found this on mongodb, but i don't know how to user it with mongoose. Is it the right thing to do?

This is the document i try to save

{ title: 'test',
  tags: 'test',
  code: 'var gulp = require(\'gulp\');\nvar source = require(\'vinyl-source-stream\');\nvar browserify = require(\'browserify\');\nvar browserSync = require(\'browser-sync\');\nvar nodemon = require(\'gulp-nodemon\');\nvar babelify = require(\'babelify\');\nvar eslint = require(\'gulp-eslint\');\nvar buffer = require(\'vinyl-buffer\');\nvar uglify = require(\'gulp-uglify\');\nvar sourcemaps = require(\'gulp-sourcemaps\');\nvar sass = require(\'gulp-ruby-sass\');\nvar config = require(\'./config\');\nvar imageop = require(\'gulp-image-optimization\');\nvar concat = require(\'gulp-concat\');\n\ngulp.task(\'images\', function(cb) {\n  gulp.src(\'./src/assets/images/**/*\')\n    .pipe( imageop({\n      optimizationLevel: 5,\n      progressive: true,\n      interlaced: true\n    }))\n    .pipe(gulp.dest(\'./public/images/\'));\n});\n\n\ngulp.task(\'browser-sync\', [\'nodemon\'], function() {\n  browserSync({\n    files: [\'public/**/*.*\'],\n    proxy: "localhost:" + config.APP_PORT,  // local node app address\n    port: 5000,  // use *different* port than above\n    notify: true,\n    browser: "google chrome",\n  });\n});\n\ngulp.task(\'nodemon\', function(cb) {\n  var called = false;\n  return nodemon({\n    script: \'./index.js\',\n    ignore: [\n      \'gulpfile.js\',\n      \'node_modules/\',\n      \'public/,\'\n    ]\n  })\n    .on(\'start\', function () {\n      if (!called) {\n        called = true;\n        cb();\n      }\n    })\n    .on(\'restart\', function () {\n      setTimeout(function () {\n        browserSync.reload({ stream: false });\n      }, 200);\n    });\n});\n\n\ngulp.task(\'browserify\', function () {\n  return browserify({entries: \'./src/client/app.js\', extensions: [\'.js\'], debug: true})\n    .transform(babelify)\n    .bundle()\n    .pipe(source(\'app.min.js\'))\n    .pipe(buffer())\n    .pipe(sourcemaps.init({loadMaps: true}))\n    // Add transformation tasks to the pipeline here.\n    .pipe(uglify())\n    .pipe(sourcemaps.write(\'./\'))\n    .pipe(gulp.dest(\'./public/js\'));\n});\n\ngulp.task(\'lint\', function (cb) {\n  return gulp.src([\'./src/**/*.js\'])\n    .pipe(eslint())\n    .pipe(eslint.format())\n    .pipe(eslint.failOnError());\n});\n\ngulp.task(\'sass\', function () {\n  return sass(\'./src/assets/sass/style.sass\', { sourcemap: true,  style: \'pressed\' })\n    .on(\'error\', sass.logError)\n    .pipe(sourcemaps.write(\'./\', {\n      includeContent: false,\n      sourceRoot: \'source\'\n    }))\n    .pipe(gulp.dest(\'./public/css/\'));\n});\n\ngulp.task(\'scripts\', function(){\n  return gulp.src([\'./src/assets/js/bootstrap.min.js\',\'./src/assets/js/checkbox.js\',\'./src/assets/js/radio.js\',\'./src/assets/js/bootstrap-switch.js\',\'./src/assets/js/toolbar.js\',\'./src/assets/js/application.js\'])\n    .pipe(sourcemaps.init())\n    .pipe(concat(\'assets.min.js\'))\n    .pipe(gulp.dest(\'./public/js/\'))\n    .pipe(uglify())\n    .pipe(sourcemaps.write(\'./\'))\n    .pipe(gulp.dest(\'./public/js/\'));\n});\n\ngulp.task(\'fonts\', function() {\n  return gulp.src(\'./src/assets/fonts/**/*\').pipe(gulp.dest(\'./public/fonts/\'));\n});\n\ngulp.task(\'watch\', function() {\n  gulp.watch([\'./src/**/*.js\'], [\'lint\', \'browserify\', \'scripts\', browserSync.reload]);\n  gulp.watch([\'./src/assets/sass/**/*.sass\'], [\'sass\', browserSync.reload]);\n});\n\ngulp.task(\'serve\', [\'browser-sync\', \'watch\']);\n\ngulp.task(\'default\', [\'browserify\', \'nodemon\', \'sass\', \'scripts\', \'fonts\', \'images\', \'watch\']);' }

Thank you very much for your help!

• javascript
• node.js
• mongodb
• mongoose

Share Follow edited Nov 18, 2015 at 15:58 Crak_mboutin asked Nov 17, 2015 at 22:08 Crak_mboutinCrak_mboutin 1,08633 gold badges1212 silver badges1818 bronze badges 4

• 2 How big is big? Mongo has a default limit of 16MB per document, so if the size of your text field and the rest of the data in the document is over 16MB Mongo will error out. – David says Reinstate Monica Commented Nov 18, 2015 at 4:28
• This is what i try to send: pastebin./GH239n3a with a simple field "title" and another "tags" who had the word "test" in it – Crak_mboutin Commented Nov 18, 2015 at 14:14
• Pastebin is blocked at my work, so I cant see it, but the question remains: exactly how big is the text, in bytes? – David says Reinstate Monica Commented Nov 18, 2015 at 14:15
• This is what i'm trying to pass github./DWboutin/Beverage/blob/master/gulpfile.js. Not in a file but in text. Thank you for your help! – Crak_mboutin Commented Nov 18, 2015 at 14:20

Add a ment  | 

2 Answers 2

Sorted by: Reset to default 5

Looks like you are trying to use the field code as your index. Based on your link the size of the code value is ~3KB. Mongo has a limitation that index fields must not exceed 1KB. 3 > 1 therefore you are getting this error.

You can fix this by either:

1. Reducing the size of the text you are entering
2. Not indexing on the code field

For index code field use text index. For unique use hash form code. For example:

import crypto  from 'crypto';

let recipeSchema = new Schema({
  title: { type: String, required: true  },
  author: { type: String},
  tags: { type: String, required: true },
  code: { type: String, required: true },
  codeHash: {type: String, unique: true },
  created_at: Date,
  updated_at: Date
});

recipeSchema.index({code: 'text'}

recipeSchema.pre('save', function( 

  var recipe = this;
  // get the current date
  var currentDate = new Date();

  // change the updated_at field to current date
  recipe.updated_at = currentDate;

  // if created_at doesn't exist, add to that field
  if (!recipe.created_at){
    recipe.created_at = currentDate;
  }

  var hash = crypto.createHash('md5').update(this.code).digest("hex")
  this.codeHash = hash;
  next();
});

本文标签： javascriptMongoose won39t save huge textStack Overflow