C API to access file in maprfs

Document created by snayeem Employee on Feb 27, 2016
Version 1Show Document
  • View in full screen mode

Author: Sanjamala Nayeem

Original Publication Date: December 22, 2014

 

C-API
Files can be directly written to and read from the MFS from C programs by using a C Application Programming Interface (API). In general, writing to or reading from MFS would require replacing the normal C open, close, read and write functions with something like the following pseudo code:

#include "hdfs.h"

int main(int argc, char **argv) {

  .

  .

  .

  hdfsFS fs = hdfsConnect("default", 0);

  .

  .

  .

  hdfsFile writeFile = hdfsOpenFile(fs, writeFileName, O_WRONLY, bufferSize, 0, 0);

  .

  .

  .

  hdfsWrite(fs, writeFile, (void*)buffer, curSize);

  .

  .

  .

  hdfsCloseFile(fs, writeFile);

  hdfsDisconnect(fs);

  return 0;

}

For reading, one would substitute O_RDONLY for O_WRONLY in the hdfsOpenFile call, and would use calls to hdfsRead to do the reads. Examples of using the C-API can be found in https://github.com/mapr/hadoop-common/tree/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test

 

Maprfs Support O_RDWR

Mapr file system support O_RDWR mode also, in hdfs does not support this mode. When file is open in O_RDWR mode alone and any operation is performed, read or write, file is truncated first and then the operation is performed, in order to retain the content in the file, one need to open file in O_RDWR|O_APPEND mode, then file is not truncated.

 

Example to use O_RDWR

#include "stdlib.h"

#include "stdio.h"

#include "string.h"

#include "hdfs.h"

int main(int argc, char **argv) {

  tOffset offset=0;

  tSize curSize;

  char* alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";

  char* numerics = "0123456789";

  if (argc != 2) {

  perror( "Usage: hdfs_read <filename> ");

  exit(-1);

  }

  int err = hdfsSetRpcTimeout(30);

  if (err) {

  perror( "Oops! Failed to set rpc timeout!");

  exit(-1);

  }

  hdfsFS fs = hdfsConnect("default", 0);

  if (!fs) {

  perror("Oops! Failed to connect to hdfs!");

  exit(-1);

  }

  const char* rfile = argv[1];

  hdfsFile writeFile = hdfsOpenFile(fs, rfile, O_RDWR, 0, 0, 0); 

  if (!writeFile) {

  printf("Failed to open %s for writing!\n", rfile);

  exit(-2);

  }

/*******************

  offset=0;

  curSize =hdfsPwrite(fs,writeFile,offset, alphabet, strlen(alphabet)+1);

  hdfsFlush(fs, writeFile);

  printf("Wrote:%s\n to file:%s at position=%lu. Characters=%d\n",alphabet,rfile,offset,curSize);

  **************/

  offset=0;

  curSize =hdfsPwrite(fs,writeFile,offset,numerics, strlen(numerics)+1);

  printf("Wrote:%s\n to file:%s at position=%lu. Characters=%d\n",numerics,rfile,offset,curSize);

  hdfsFlush(fs, writeFile);

/********************

  offset=20;

  curSize = hdfsPwrite(fs,writeFile,offset,alphabet, strlen(alphabet)+1);

  printf("Wrote:%s\n to file:%s at position=%lu. Characters=%d\n",alphabet,rfile,offset,curSize);

  ************************/

  hdfsFlush(fs, writeFile);

  hdfsCloseFile(fs, writeFile);

  hdfsDisconnect(fs);

}

Example of O_RDWR | O_APPEND

 

#include "stdlib.h"

#include "stdio.h"

#include "string.h"

#include "hdfs.h"

 

int main(int argc, char **argv) {

  tOffset offset=0;

  tSize curSize;

  char* alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";

  char* numerics = "0123456789";

  if (argc != 2) {

  perror( "Usage: hdfs_read <filename> ");

  exit(-1);

  }

  int err = hdfsSetRpcTimeout(30);

  if (err) {

  perror( "Oops! Failed to set rpc timeout!");

  exit(-1);

  }

  hdfsFS fs = hdfsConnect("default", 0);

  if (!fs) {

  perror("Oops! Failed to connect to hdfs!");

  exit(-1);

  }

  const char* rfile = argv[1];

  hdfsFile writeFile = hdfsOpenFile(fs, rfile, O_RDWR | O_APPEND, 0, 0, 0); 

  if (!writeFile) {

  printf("Failed to open %s for writing!\n", rfile);

  exit(-2);

  }

/*******************

  offset=0;

  curSize =hdfsPwrite(fs,writeFile,offset, alphabet, strlen(alphabet)+1);

  hdfsFlush(fs, writeFile);

  printf("Wrote:%s\n to file:%s at position=%lu. Characters=%d\n",alphabet,rfile,offset,curSize);

  **************/

  offset=0;

  curSize =hdfsPwrite(fs,writeFile,offset,numerics, strlen(numerics)+1);

  printf("Wrote:%s\n to file:%s at position=%lu. Characters=%d\n",numerics,rfile,offset,curSize);

  hdfsFlush(fs, writeFile);

/********************

  offset=20;

  curSize = hdfsPwrite(fs,writeFile,offset,alphabet, strlen(alphabet)+1);

  printf("Wrote:%s\n to file:%s at position=%lu. Characters=%d\n",alphabet,rfile,offset,curSize);

  ************************/

  hdfsFlush(fs, writeFile);

  hdfsCloseFile(fs, writeFile);

  hdfsDisconnect(fs);

}

 

To compile and execute the code following instructions in the bellow linkhttp://doc.mapr.com/display/MapR/Accessing+MapR-FS+in+C+Applications

Attachments

    Outcomes