source: tags/1.0/disparm.cpp @ 5

Last change on this file since 5 was 5, checked in by willem, 11 years ago

willem

File size: 20.8 KB
Line 
1#include <iostream>
2#include <fstream>
3#include <libgen.h>
4#include <cstdlib>
5#include <string.h>
6
7#include "disparm.h"
8#include "tofromstring.h"
9#include "token.h"
10#include "shellenv.h"
11#include <sys/stat.h>
12#include <errno.h>
13#include <dirent.h>
14
15const std::string disparm::keyname       = "DISPARM_KEY";
16const std::string disparm::committedname = "DISPARM_COMMITTED";
17const std::string disparm::valuename     = "DISPARM_VALUE";
18const std::string disparm::rcname        = "DISPARM_RC";
19const std::string disparm::linename      = "DISPARM_LINENO";
20
21const std::string disparm::defpath_pool   = "pool.disparm";
22const std::string disparm::definputfilename = "-";
23
24//
25//constructor disparm
26//
27disparm::disparm()
28{
29  char defpoolname_c[this->defpath_pool.length()+1];        // default path_pool
30  strcpy(defpoolname_c,this->defpath_pool.c_str());         
31  //
32  // A note about basename and dirname: they need as argument a char*
33  // NOT a const char*. Both functions may modify the argument,
34  // so, in general, do not use the same argument more than
35  // once.
36  //
37  char defpath_lockdir_c[this->defpath_lockdir.length()+1];  // default directory for the lockfile
38  strcpy(defpath_lockdir_c,this->defpath_lockdir.c_str());
39  this->setpath_pool(this->defpath_pool);
40  this->defpath_lockdir = std::string(dirname(defpoolname_c));
41  this->setpath_link("");
42  this->setpath_lock("");     // the name of the lock folder
43  this->setbaselinkfilename("");
44  this->setinputfilename(this->definputfilename);  // default input file name
45  this->setpath_lockdir(this->defpath_lockdir);      // default lock directory
46  this->setmaxiter(1);                             // default maximum times a string can be given out
47  this->pmarker="pp";                              // start of record with pointer to next string in pool
48  this->pmarkerlen = this->pmarker.length();
49  this->lockname="lock.disparm";
50
51  std::string shell = getenv("SHELL");
52  if (shell.find("csh") != std::string::npos) 
53    this->shelltype = SHELL_CSH;
54  else
55    this->shelltype = SHELL_SH;
56
57  //
58  // following items are returned on stdout as in:
59  // export DISPARM_KEY=1234
60  // See also above
61  //
62  this->key       = -1;     // key: pointer to next item in pool
63  this->status    = '0';
64  this->committed = 0;      // the number of times the string is given out
65  this->rc        = "OK";   // return code
66  this->value     = "";     // the string to give out
67  this->linenr    = 0;      // the line number in the pool
68
69  this->debug     = 0;
70}
71
72void disparm::removelock()
73{
74  rmdir(this->path_lock.c_str());
75  unlink(this->path_link.c_str());
76}
77
78void disparm::setpath_pool(std::string pool)
79{
80  this->path_pool = pool;
81}
82
83void disparm::setpath_link(std::string path_link)
84{
85  this->path_link = path_link;
86}
87
88void disparm::setpath_lock(std::string path_lock)
89{
90  this->path_lock = path_lock;
91}
92
93void disparm::setbaselinkfilename(std::string baselinkfilename)
94{
95  this->baselinkfilename = baselinkfilename;
96}
97
98int disparm::setpath_lockdir(std::string lockdir)
99{
100  this->path_lockdir = lockdir;
101  int rc = mkdir(path_lockdir.c_str(),0777);
102  if (rc ==0)
103    return 0;
104  else
105  {
106    if (errno == EEXIST)
107    {
108      struct stat buf;
109      stat(path_lockdir.c_str(),&buf);
110      if (S_ISDIR(buf.st_mode))
111        return 0;
112    }
113    else
114      return -1;
115  }
116  return 0;  // never reached
117}
118
119//
120// we try to generate a name for a linkfile that is unique.
121// In the name is also the creation time.
122// We also create a suitable path_lockdir here.
123//
124void disparm::createlinkfilename()
125{
126  if (this->debug)
127    this->debugger("Before createlinkfilename:");
128  char hostname[1025];
129  gethostname(hostname,1024);
130  char poolname_c[this->path_pool.length()+1];
131  strcpy(poolname_c,this->path_pool.c_str());
132  std::string poolnamebase = basename(poolname_c);
133  std::string base = this->path_lockdir + "/" + poolnamebase + ".link.";
134  this->setbaselinkfilename(base);
135  this->setpath_link(   base
136                         + to_string(time(0))
137                         + "." 
138                         + hostname
139                         + "." 
140                         + to_string(getpid()));
141  this->setpath_lock(this->path_lockdir + "/" + this->lockname);
142  if (this->debug)
143    this->debugger("After createlinkfilename:");
144}
145
146void disparm::setinputfilename(std::string inputfilename)
147{
148  this->inputfilename = inputfilename;
149}
150
151void disparm::setkey(std::streampos key)
152{
153  this->key = key;
154}
155
156std::streampos disparm::getkey(void)
157{
158  return this->key;
159}
160
161void disparm::setmaxiter(int maxiter)
162{
163  this->maxiter = maxiter;
164}
165
166//
167// Create poolfile from input file
168//
169// On return, s wil contain the output of createrc
170//
171int disparm::create(std::string &s)
172{
173  if (this->debug)
174    this->debugger("Before create:");
175  std::ifstream inp;
176  std::istream *pinp;
177  s = "";
178  //
179  // open inputfile, take care that stdin is also allowed
180  //
181  if (this->inputfilename == "-")
182    pinp = &std::cin;
183  else
184  {
185    inp.open(this->inputfilename.c_str());
186    if (!inp.is_open())
187    {
188      std::cerr<<"Cannot open '"<<inputfilename<<"'"<<std::endl;
189      this->rc = "FAIL";
190      s = this->createrc();
191      return 1;
192    }
193    pinp = &inp;
194  }
195  //
196  // open poolfile
197  //
198  std::string line;
199  std::fstream out;
200  out.open(this->path_pool.c_str(),std::fstream::in | std::fstream::out
201                               |  std::fstream::trunc);
202
203  //
204  // fill poolfile
205  //
206  token tok;
207  tok.setfile(out);
208  std::streampos p;
209  unsigned int linenr = 0;
210  while (getline(*pinp,line))
211  {
212    tok.setlinenr(++linenr);
213    tok.setcommitted(0);
214    tok.clearstatus();
215    tok.setvalue(line);
216    tok.writetoken(p);
217  }
218  tok.writelocpointer(0);
219
220  //
221  // close input file, only if it is not stdin
222  //
223  if (pinp == &inp)
224    inp.close();
225  out.close();
226
227  this->key = 0;
228  s = createrc();
229  if (this->debug)
230    this->debugger("After create:");
231
232  return 0;
233}
234
235//
236// get next string from pool
237//
238int disparm::next(std::string& s)
239{
240  if (this->debug)
241    this->debugger("Before next:");
242  int rc = this->realnext();
243  s = this->createrc();
244  if (this->debug)
245    this->debugger("After next:");
246  return rc;
247}
248
249//
250// create string to be eval'd by the shell
251//
252std::string disparm::createrc(void)
253{
254  std::string s = "";
255  s = s + shellenv(to_string(this->key),      this->keyname,      this->shelltype);
256  s = s + shellenv(to_string(this->committed),this->committedname,this->shelltype);
257  s = s + shellenv(to_string(this->value),    this->valuename,    this->shelltype);
258  s = s + shellenv(to_string(this->rc),       this->rcname,       this->shelltype);
259  s = s + shellenv(to_string(this->linenr),   this->linename,     this->shelltype);
260  return s;
261}
262
263//
264// get the next string from the pool
265//
266int disparm::realnext()
267{
268  std::fstream file;
269  bool good = 1;
270
271  this->key       = 0;
272  this->status    = '0';
273  this->committed = 0;
274  this->rc        = "OK";
275  this->linenr    = 0;
276  this->value     = "";
277
278  //
279  // get lock, return if that fails
280  //
281  if (this->getlock()!=0)
282  {
283    this->rc = "FAIL";
284    return 1;
285  }
286
287  //
288  // read next string from pool
289  // take care of ready records, maximum iterations
290  // allowed and some kinds of mishaps
291  //
292  file.open (this->path_link.c_str(), std::fstream::in | std::fstream::out);
293  if (file.is_open())
294  {
295    token       tok;
296    std::string line;
297    tok.setfile(file);
298    std::streampos locpointer, posused, posnext;
299
300    //
301    // get pointer to next string
302    //
303    locpointer = tok.readlocpointer();
304    if (locpointer < 0)
305    {
306      std::cerr << "Corrupt parameterfile: cannot find locpointer\n" << std::endl;
307      this->rc = "FAIL";
308      good = 0;
309    }
310    //
311    // read next string from pool
312    //
313    if (good)
314    {
315      int rc = tok.readnext(locpointer,posused,posnext);
316      if (rc != 0)
317      {
318        std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
319        std::cerr << "Corrupt or empty parameterfile: cannot read next line\n" << std::endl;
320        this->rc = "FAIL";
321        good = 0;
322      }
323    }
324    //
325    // keep reading until a valid record is found
326    // take appropriate action if no such record exists
327    //
328    if (good)
329    {
330      std::streampos locpointerold = locpointer;
331      bool allready = 1;  // to learn if all records are
332                          // ready
333      while (1)
334      {
335        if (!tok.getready())
336        {
337          allready = 0;
338          if (tok.getcommitted() >= this->maxiter)
339          {
340            good = 0;
341            this->rc = "MAX_EXCEEDED";
342            break;
343          }
344          this->key = posused;
345          this->status = tok.getstatus();
346          this->committed = tok.getcommitted();
347          this->value = tok.getvalue();
348          this->linenr = tok.getlinenr();
349          tok.setcommitted(tok.getcommitted()+1);
350          tok.writetoken(posused);
351          tok.writelocpointer(posused);
352          break;
353        }
354        locpointer = posnext;
355        if (locpointer == locpointerold)
356        {
357          //
358          // so we are cycling, nothing to find, it seems
359          //
360          good = 0;
361          if (allready)
362            this->rc = "ALL_READY";
363          else
364            this->rc = "MAX_EXCEEDED";
365          break;
366        }
367        int rc = tok.readnext(locpointer,posused,posnext);
368        if (rc != 0)
369        {
370          std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
371          std::cerr << "Corrupt or empty parameterfile: cannot read next line\n" << std::endl;
372          this->rc = "FAIL";
373          good = 0;
374          break;
375        }
376      }
377    }
378  }
379  else
380  {
381    std::cerr<<"Strange error: cannot open '"<<path_link<<"'"<<std::endl;
382    return 1;
383  }
384
385  file.close();
386  this->removelock();
387  return !good;
388}
389
390//
391// get a lock on the poolfile
392// The normal methods don't work on an nfs filesystem.
393// Luckily, there is a method based on links to the
394// file to be locked, see also man(2) open under O_EXCL
395//
396// Creating a link is atomic, so it suffices to see if
397// the link count equals to two. If not, we try again.
398//
399// It seems that the file used for linking contains displays
400// an up-to-date content.
401//
402int disparm::getlock(void)
403{
404  struct stat buf;
405  int rc;
406  int sleeptime=2;
407
408  int iter = 0;
409  int ntryremovelock = 0;
410  while(1)
411  {
412    //
413    // check poolfile's linkcount
414    // if link count == 1, we try to create a link, else wait
415    //
416    rc  = stat(this->path_pool.c_str(),&buf);
417    std::string er="";
418    switch (errno)
419    {
420      case (EACCES): er="Search permission is denied for one of the  directories  in  the path prefix of path.  (See also path_resolution(7).)";
421                     break;
422
423      case(EBADF):  er="fd is bad.";
424                    break;
425
426      case(EFAULT): er="Bad address.";
427                    break;
428
429      case(ELOOP):  er="Too many symbolic links encountered while traversing the path.";
430                    break;
431
432      case(ENAMETOOLONG):er="File name too long.";
433                         break;
434
435      case(ENOENT): er="A component of path does not exist, or path is an empty string.";
436                    break;
437
438      case(ENOMEM): er="Out of memory (i.e., kernel memory).";
439                    break;
440
441      case(ENOTDIR):
442              er="A component of the path prefix of path is not a directory.";
443              break;
444
445      case(EOVERFLOW):er="(stat())  path refers to a file whose size cannot be represented in the type off_t.  This can occur when an application  compiled on a 32-bit platform without -D_FILE_OFFSET_BITS=64 calls stat() on a file whose size exceeds (2<<31)-1 bits";
446                      break;
447
448    }
449    if (rc !=0 )
450    {
451      std::cerr<<er<<std::endl;
452      perror(" Error stat poolfile:");
453      std::cerr<<"Cannot stat the pool file:'"<<this->path_pool.c_str()<<"'"<< std::endl;
454      return 1;
455    }
456    this->createlinkfilename();
457    rc = mkdir(this->path_lock.c_str(), 0700);
458    if (rc == 0)
459    {
460      // create link
461      int rc = link(this->path_pool.c_str(),this->path_link.c_str());
462      if (rc < 0)
463      {
464        std::cerr<<"Cannot make link '"<<this->path_link<<"' to '"
465          <<this->path_pool<<"'"<<std::endl<<" Is poolname '"<<path_pool
466          <<"' correct?"<<std::endl;
467        return 1;
468      }
469      break;
470    }
471    //
472    // wait some time before trying again
473    //
474    iter++;
475    std::cerr << iter <<": waiting for lock " << std::endl;
476    sleep(sleeptime);
477    if (iter > 9)
478    {
479      std::cerr << "Trying to remove locks" << std::endl;
480      //
481      // remove stale link files
482      //
483      this->remove_lock_and_links();
484      ntryremovelock ++;
485      if (ntryremovelock > 10)
486      {
487        std::cerr << "Cannot get lock after many tries." << std::endl;
488        return 1;
489      }
490      iter = 0;
491    }
492  }
493  return 0;
494}
495
496//
497// mark string as ready
498//
499int disparm::ready()
500{
501  if (this->getlock()!=0)
502  {
503    return 1;
504  }
505
506  //
507  // if no key given as parameter, read it from environment
508  //
509  if (this->key < 0)
510  {
511    char * s = getenv(this->keyname.c_str());
512    if (s == NULL || *s == 0)
513    {
514      this->removelock();
515      std::cerr << "No key given" << std::endl;
516      return 1;
517    }
518    else
519    {
520      this->key = from_string<long long>(std::string(s));
521    }
522  }
523
524  std::streampos locpointer = this->key;
525  std::fstream file;
526
527  file.open (this->path_link.c_str(), std::fstream::in | std::fstream::out);
528  if (file.is_open())
529  {
530    token tok;
531    std::string line;
532    tok.setfile(file);
533    std::streampos posused,posnext;
534    //
535    // read next record
536    //
537    int rc = tok.readnext(locpointer,posused,posnext);
538    if (rc != 0)
539    {
540      std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
541      std::cerr << "Corrupt or empty parameterfile: cannot read next line\n" << std::endl;
542      file.close();
543      this->removelock();
544      return 1;
545    }
546    //
547    // mark record as ready
548    //
549    tok.ready();
550    //
551    // write record back
552    //
553    tok.writetoken(locpointer);
554  }
555  file.close();
556  this->removelock();
557  return 0;
558}
559
560//
561// remove poolfile
562//
563int disparm::REMOVE()
564{
565  if (this->getlock()!=0)
566  {
567    return 1;
568  }
569  this->removelock();
570
571  int rc = unlink(this->path_pool.c_str());
572  if (rc)
573  {
574    std::cerr << "Cannot remove " << this->path_pool << std::endl;
575    return 1;
576  }
577  return 0;
578}
579
580//
581// return statistics about the pool
582// no wait for lock, so be careful interpretating the results
583//
584stats disparm::getstats()
585{
586  stats statistics;
587  token tok;
588  std::fstream file;
589  statistics.records   = -1;
590  statistics.committed = -1;
591  statistics.ready     = -1;
592  statistics.commax    = -1;
593
594
595  file.open (this->path_pool.c_str(), std::fstream::in | std::fstream::out);
596
597  if (file.is_open())
598  {
599    tok.setfile(file);
600    std::streampos pos = 0;
601    std::streampos posused;
602    std::streampos posnext;
603    statistics.records   = 0;
604    statistics.committed = 0;
605    statistics.ready     = 0;
606    statistics.commax    = 0;
607   
608    while(1)
609    {
610      tok.readnext(pos, posused,posnext);
611      if (pos != posused)
612        break;
613      if (tok.getready())
614        statistics.ready++;
615      if (tok.getcommitted()>0)
616        statistics.committed++;
617      if (tok.getcommitted() > statistics.commax)
618        statistics.commax = tok.getcommitted();
619      statistics.records++;
620      pos = posnext;
621    }
622  }
623
624  file.close();
625  return statistics;
626}
627
628int disparm::echounhandled(void)
629{
630  token tok;
631  std::fstream file;
632
633  file.open (this->path_pool.c_str(), std::fstream::in | std::fstream::out);
634
635  if (file.is_open())
636  {
637    tok.setfile(file);
638    std::streampos pos = 0;
639    std::streampos posused;
640    std::streampos posnext;
641    std::cerr << "linenr:committed:value"<< std::endl;
642   
643    while(1)
644    {
645      tok.readnext(pos, posused,posnext);
646      if (pos != posused)
647        break;
648
649      if (!tok.getready()&&tok.getcommitted()>0)
650      {
651        std::cerr << tok.getlinenr() << ":"<<tok.getcommitted()<<":" << tok.getvalue() << std::endl;
652      }
653      pos = posnext;
654    }
655  }
656  else
657  {
658    std::cerr <<"Cannot open pool file '"<<this->path_pool<<"'"<<std::endl;
659    return 1;
660  }
661
662  file.close();
663  return 0;
664}
665
666
667int disparm::remove_lock_and_links(const std::string path, const std::string dirl, const std::string path_lock, int timer)
668{
669  // removes lock and hard links in directory "dirl" to file "path"
670  // in directory "dir", if they are older than "time" seconds.
671  // Method:
672  // if lock is older than timer seconds, remove it.
673  // and:
674  // using readdir, examine all components of dirl and see
675  // if they have the same inode as "path". If so, remove, unless
676  // the realname is the same as the realname of "path"
677  //
678  DIR *pdir;
679  struct dirent *pent;
680  int rc;
681  struct stat buf;
682
683  pdir = opendir(dirl.c_str());
684  if (!pdir)
685  {
686    std::cerr << "Cannot read link directory '"<<dirl<<"'"<<std::endl;
687    return 1;
688  }
689  // look if lock can be removed, if yes: and make it so
690  rc = stat(path_lock.c_str(),&buf);
691  if (rc == 0)
692  {
693    if (time(0) > buf.st_mtime + timer)
694    {
695      std::cerr << "Trying to remove '"<<path_lock<<"'"<<std::endl;
696      rc = rmdir(path_lock.c_str());
697      // It could happen that rmdir failed, for example another process
698      // already deleted it, or path is invalid etc.
699      // We ignore this, relying on the fact that eventually
700      // getlock will time out.
701    }
702  }
703  rc = stat(path.c_str(),&buf);
704  if (rc !=0)
705  {
706    std::cerr << "Cannot stat '"<<path<<"'"<<std::endl;
707    return 1;
708  }
709  ino_t inodenr = buf.st_ino;
710  char* rpathc = (char*)malloc(PATH_MAX+1);
711  rpathc = realpath(path.c_str(),rpathc);
712  std::string rpath = rpathc;
713  std::string rpath1;
714 
715  while ((pent=readdir(pdir)))
716  {
717    struct stat buf;
718    std::string name = pent->d_name;
719    if (name == ".") continue;
720    if (name == "..") continue;
721    rpathc = realpath((dirl+"/"+name).c_str(),rpathc);
722    rpath1 = rpathc;
723    std::cerr<<"TD: looking at '"<<rpath1<<"'"<<std::endl;
724    if (rpath != rpath1)
725    {
726      int rc = stat(rpath1.c_str(),&buf);
727      if (rc != 0)
728      {
729        std::cerr<<"Cannot stat '"<<rpath1<<"'"<<" this cannot happen!" << std::endl;
730        free(rpathc);
731        return 1;
732      }
733      if (buf.st_ino == inodenr)
734      {
735        std::cerr << "time: " << time(0) << " " << buf.st_mtime << " " << timer << std::endl;
736        if (time(0) > buf.st_mtime + timer)
737        {
738          int rc = unlink(rpath1.c_str());
739          if (rc == 0)
740            std::cerr<<"Unlinked '"<<rpath1<<"'"<<std::endl;
741          else
742            std::cerr<<"Could not unlink '"<<rpath1<<"'"<<std::endl;
743        }
744        else
745          std::cerr << "TD: not unlinking because time: '"<<rpath1<<"'"<<std::endl;
746      }
747    }
748    else
749      std::cerr<<"TD: not unlinking '"<<rpath1<<"'"<<std::endl;
750  }
751  free(rpathc);
752  return 0;
753}
754
755
756//
757// remove lock and links
758// method: do a glob on baselinkfilename and unlink
759// the files.
760//
761// Only remove if links are older than 60 seconds.
762//
763void disparm::remove_lock_and_links()
764{
765  if (this->debug)
766    this->debugger("Before remove_lock_and_links:");
767
768  remove_lock_and_links(this->path_pool,this->path_lockdir,this->path_lock,60);
769  if (this->debug)
770    this->debugger("Before remove_lock_and_links:");
771}
772
773void disparm::setdebug()
774{
775  this->debug = 1;
776}
777void disparm::debugger(const std::string s)
778{
779  std::cerr << s << std::endl;
780  std::cerr << "baselinkfilename: '" << this->baselinkfilename << "'" << std::endl;
781  std::cerr << "committed:        '" << this->committed << "'" << std::endl;
782  std::cerr << "committedname:    '" << this->committedname << "'" << std::endl;
783  std::cerr << "definputfilename: '" << this->definputfilename << "'" << std::endl;
784  std::cerr << "defpath_pool:     '" << this->defpath_pool << "'" << std::endl;
785  std::cerr << "defpath_lockdir:  '" << this->defpath_lockdir << "'" << std::endl;
786  std::cerr << "inputfilename:    '" << this->inputfilename << "'" << std::endl;
787  std::cerr << "key:              '" << this->key << "'" << std::endl;
788  std::cerr << "keyname:          '" << this->keyname << "'" << std::endl;
789  std::cerr << "linename:         '" << this->linename << "'" << std::endl;
790  std::cerr << "linenr:           '" << this->linenr << "'" << std::endl;
791  std::cerr << "lockname:         '" << this->lockname << "'" << std::endl;
792  std::cerr << "maxiter:          '" << this->maxiter << "'" << std::endl;
793  std::cerr << "path_link:        '" << this->path_link << "'" << std::endl;
794  std::cerr << "path_lock:        '" << this->path_lock << "'" << std::endl;
795  std::cerr << "path_lockdir:     '" << this->path_lockdir << "'" << std::endl;
796  std::cerr << "path_pool:         '" << this->path_pool << "'" << std::endl;
797  std::cerr << "pmarker:          '" << this->pmarker << "'" << std::endl;
798  std::cerr << "pmarkerlen:       '" << this->pmarkerlen << "'" << std::endl;
799  std::cerr << "pointerlen:       '" << this->pointerlen << "'" << std::endl;
800  std::cerr << "rc:               '" << this->rc << "'" << std::endl;
801  std::cerr << "rcname:           '" << this->rcname << "'" << std::endl;
802  std::cerr << "shelltype:        '" << this->shelltype << "'" << std::endl;
803  std::cerr << "status:           '" << this->status << "'" << std::endl;
804  std::cerr << "value:            '" << this->value << "'" << std::endl;
805  std::cerr << "valuename:        '" << this->valuename << "'" << std::endl;
806}
807
808//
809// Just to teach me again how to make an object
810// suitable for the << operator
811//
812std::ostream& operator <<(std::ostream &os,disparm &obj)
813{
814  stats sta;
815  sta = obj.getstats();
816  os << "Total:     " << sta.records   << "\n" 
817     << "Committed: " << sta.committed << "\n"
818     << "Ready:     " << sta.ready     << "\n"
819     << "Max:       " << sta.commax;
820  return os;
821}
822
Note: See TracBrowser for help on using the repository browser.