javascript - Creating a Readable Stream to use with FormData - Stack Overflow

admin管理员组

文章数量:1398999

I'm using an API to upload a CSV file. I create the CSV file in memory from a String and upload it using the request module. However, I'm having trouble creating the Readable Stream from the String. I followed a SO answer on How to create streams from string in Node.Js. Here is my code for that solution:

var importResponse = function(csv, callback){
    stringify(csv, function(err, output){

        const s = new Readable();
        s._read = () => {}; 
        s.push(output);
        s.push(null);

        request.post({
          headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
          url: '',
          formData: {
            surveyId: 'SV_123',
            file: {
                value: s,
                options: {
                    contentType: 'text/csv; charset=utf-8'
                }
            }
          }
        }, function(err, res, body){
            if(err || res.statusCode !== 200){
              console.log(err || "Error status code: " + res.statusCode);
              console.log(body);
              return;
            }
        });
    });

}

The csv variable looks like [["QID1","QID2"],["1","2"]] and the output from stringify looks like "QID1,QID2\n,1,2\n".

This solution gives me the error Unexpected end of input

{"meta":{"httpStatus":"400 - Bad Request","error":{"errorMessage":"Unexpected end of input"}}}

If instead I use memfs, it works fine

const fs = require('memfs');

var importResponse = function(csv, callback){
    stringify(csv, function(err, output){
        // Create file in memory
        fs.writeFileSync('/data.csv', output); 

        request.post({
          headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
          url: '',
          formData: {
            surveyId: 'SV_123',
            file: {
                value: fs.createReadStream('/data.csv'),
                options: {
                    contentType: 'text/csv; charset=utf-8'
                }
            }
          }
        }, function(err, res, body){
            if(err || res.statusCode !== 200){
              console.log(err || "Error status code: " + res.statusCode);
              console.log(body);
              return;
            }
        });
    });

}

How can I convert the output from stringify to a Stream that I can use to upload via the api?

I'm using an API to upload a CSV file. I create the CSV file in memory from a String and upload it using the request module. However, I'm having trouble creating the Readable Stream from the String. I followed a SO answer on How to create streams from string in Node.Js. Here is my code for that solution:

var importResponse = function(csv, callback){
    stringify(csv, function(err, output){

        const s = new Readable();
        s._read = () => {}; 
        s.push(output);
        s.push(null);

        request.post({
          headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
          url: 'https://ca1.qualtrics./API/v3/responseimports',
          formData: {
            surveyId: 'SV_123',
            file: {
                value: s,
                options: {
                    contentType: 'text/csv; charset=utf-8'
                }
            }
          }
        }, function(err, res, body){
            if(err || res.statusCode !== 200){
              console.log(err || "Error status code: " + res.statusCode);
              console.log(body);
              return;
            }
        });
    });

}

The csv variable looks like [["QID1","QID2"],["1","2"]] and the output from stringify looks like "QID1,QID2\n,1,2\n".

This solution gives me the error Unexpected end of input

{"meta":{"httpStatus":"400 - Bad Request","error":{"errorMessage":"Unexpected end of input"}}}

If instead I use memfs, it works fine

const fs = require('memfs');

var importResponse = function(csv, callback){
    stringify(csv, function(err, output){
        // Create file in memory
        fs.writeFileSync('/data.csv', output); 

        request.post({
          headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
          url: 'https://ca1.qualtrics./API/v3/responseimports',
          formData: {
            surveyId: 'SV_123',
            file: {
                value: fs.createReadStream('/data.csv'),
                options: {
                    contentType: 'text/csv; charset=utf-8'
                }
            }
          }
        }, function(err, res, body){
            if(err || res.statusCode !== 200){
              console.log(err || "Error status code: " + res.statusCode);
              console.log(body);
              return;
            }
        });
    });

}

How can I convert the output from stringify to a Stream that I can use to upload via the api?

• javascript
• node.js

Share Improve this question Follow asked Jul 18, 2018 at 23:26 EricEric 2,09633 gold badges2020 silver badges3636 bronze badges 3

• Just so we know for sure, what is stringify ing from? Do you have a specific package you're requiring? – Jacob Commented Jul 19, 2018 at 0:17
• 1 It looks like csv-stringify can create streams itself, FYI. You can probably just pass stringify(csv) directly as the "file" stream. – Jacob Commented Jul 19, 2018 at 0:54
• @Jacob yep, this fixed it. Changed my code to var output = stringify(csv); and it works now. Also changed the stringify module to the synchronous one. Thank you! – Eric Commented Jul 19, 2018 at 1:21

Add a ment  | 

2 Answers 2

Sorted by: Reset to default 1

It looks like you're using the request library. You may be ing across this caveat as documented in their readme:

// Pass optional meta-data with an 'options' object with style: {value: DATA, options: OPTIONS}
// Use case: for some types of streams, you'll need to provide "file"-related information manually.
// See the `form-data` README for more information about options: https://github./form-data/form-data
custom_file: {
  value:  fs.createReadStream('/dev/urandom'),
  options: {
    filename: 'topsecret.jpg',
    contentType: 'image/jpeg'
  }
}

Since you're using a non-file stream, simply providing a dummy filename should work:

request.post({
  headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
  url: 'https://ca1.qualtrics./API/v3/responseimports',
  formData: {
    surveyId: 'SV_123',
    file: {
      value: s,
      options: {
        contentType: 'text/csv; charset=utf-8',
        filename: 'dummy.csv'
      }
    }
  }
}, function(err, res, body){
  if(err || res.statusCode !== 200){
    console.log(err || "Error status code: " + res.statusCode);
    console.log(body);
    return;
  }
});

The sample snippet is incorrect or possibly outdated for current node versions. A really easy way to implement your readable:

const s = new Readable({
  encoding: 'utf8',
  read(size) {
    // Possibly respect the requested size to make for a good consumer experience
    // Otherwise:
    this.push(output, 'utf8');
    this.push(null); // This signals that there's no more data.
  }
});

Here's how you can respect the wishes of the reader:

let data = output;
const s = new Readable({
  encoding: 'utf8',
  read(size) {
    let wantsMore = true;
    while (wantsMore) {
      const chunk = data.slice(0, size);          
      if (!chunk) {
        return void this.push(null);            
      }

      wantsMore = this.push(chunk, 'utf8');
      data = data.slice(size);
    }
  }
});

本文标签： javascriptCreating a Readable Stream to use with FormDataStack Overflow