How to add the support of the get_next_object() callback to the 5-c_stats example?

To clarify, a get_next_object() that doesn’t necessarily need find_next_object() and optimize the memory and CPU resources needed by chunking the confd_data_reply_next_object_arrays() could be implemented like this:

/* Keypath example */
/* /arpentries/arpe */
/*    2         1   */

static int max_nobjs = 100; // Chunk size hardcoded here to simplify

static int get_next_object(struct confd_trans_ctx *tctx,
			     confd_hkeypath_t *keypath, long next)
{
  int pos, i;
  confd_value_t *v;
  struct confd_next_object *obj;
  struct arpdata *dp = tctx->t_opaque;
  int n_list_entries;
  struct aentry *ae;

  if (need_arp(dp)) {
    if (run_arp(dp) == CONFD_ERR)
      return CONFD_ERR;
  }

  n_list_entries = dp->n_ae;
  ae = dp->arp_entries;
  
  if (next == -1) {
    pos = 0; /* ConfD wants to get the whole table / list from the beginning */
  } else if (next < n_list_entries) {
    pos = next; /* ConfD wants the table / list from a specific index */
  } else { /* next == n_list_entries, return nothing */
    confd_data_reply_next_object_array(tctx, NULL, -1, -1);
    return CONFD_OK;
  }

  /* If we have more than say 100 objects for smaller objects to return, we chunk the reply to ConfD in 100 objects at a time for best possible performance.       
  */
  if (n_list_entries - pos > max_nobjs) {
    nobj = max_nobjs + pos;
  } else {
    nobj = n_list_entries;
  }

  for(i = 0; i < pos; i++)
    ae = ae->next;
  
  obj = malloc(sizeof(struct confd_next_object) * (nobj + 2));
  v = (confd_value_t *) malloc(sizeof(confd_value_t) * nobj * N_LEAFS);

  for (i = 0; pos + i < nobj; i++) { /* Collect all the rows in the table / list entries */
    obj[i].v = &v[i * N_LEAFS];

    CONFD_SET_IPV4(&(obj[i].v[0]), ae->ip4);
    CONFD_SET_STR(&(obj[i].v[1]), ae->iface);
    if (ae->hwaddr == NULL) {
      CONFD_SET_NOEXISTS(&(obj[i].v[2]));
    } else {
      CONFD_SET_STR(&(obj[i].v[2]), ae->hwaddr);
    }
    CONFD_SET_BOOL(&(obj[i].v[3]), ae->perm);
    CONFD_SET_BOOL(&(obj[i].v[4]), ae->pub);
    obj[i].n = N_LEAFS;
    /* If we pass -1 here we will invoke find_next_object. If we instead pass next object ConfD will invoke get_next_object if there are more objects to be fetched
    */
    obj[i].next = pos+i+1; //-1;
    ae=ae->next;
  }
  if (pos + i == n_list_entries)
    obj[i++].v = NULL;
  confd_data_reply_next_object_arrays(tctx, obj, i, 0); /* Reply with an array of object arrays. I.e. the whole table / list */
  free(v);
  free(obj);
  return CONFD_OK;
}

Why we need to apply nobj+2 , i think it should be nobj+1, the tv of last instance is filled with null as end indication.

Why i get error: non-unique key values: {01:02:03:04:05:02} ?

Correct, it should say “1” there. Thanks!

Don’t know with that little info on the error you get. Check the run_arp() function of the example and verify that it parses the ARP table in your system correctly after calling “arp -an”

I have been wanting to provide a tag value array variant to this post for a while now as I belive it is more easy to follow than a plain value array. It does the same thing but use the tag too. The memory and performance overhead of using tag_value instead of value is negligible here, while the advantage is slightly improved readability:

// NOTE: See the original post for the other changes needed to the original (here ConfD-6.7.1) arpstat.c file 
...
static int navals;
static int max_nobjs = 100; // Chunk size hardcoded here to simplify

static int get_next_object_tag(struct confd_trans_ctx *tctx,
			     confd_hkeypath_t *keypath, long next)
{
  int i;
  confd_tag_value_t *tv;
  struct confd_tag_next_object *tobj;
  struct arpdata *dp = tctx->t_opaque;
  struct aentry *curr;

  if (next == -1) {  /* first call */
    if (need_arp(dp)) {
      if (run_arp(dp) == CONFD_ERR)
	return CONFD_ERR;
    }
    curr = dp->arp_entries;
  } else {
    curr = (struct aentry *)next;
  }
  if (curr == NULL) {
    confd_data_reply_next_key(tctx, NULL, -1, -1);
    return CONFD_OK;
  }

  tobj = malloc(sizeof(struct confd_tag_next_object) * (max_nobjs + 1));
  tv = (confd_tag_value_t *) malloc(sizeof(confd_tag_value_t) * max_nobjs * navals);
  
  for (i = 0; curr != NULL && i < max_nobjs; curr = curr->next, i++) { /* Collect max_nobjs or as many as there is rows in the table / list entries */
    tobj[i].tv = &tv[i * navals];

    CONFD_SET_TAG_IPV4(&(tobj[i].tv[0]), arpe_ip, curr->ip4);
    CONFD_SET_TAG_STR(&(tobj[i].tv[1]), arpe_ifname, curr->iface);
    if (curr->hwaddr == NULL) {
      CONFD_SET_TAG_NOEXISTS(&(tobj[i].tv[2]), arpe_hwaddr);
    } else {
      CONFD_SET_TAG_STR(&(tobj[i].tv[2]), arpe_hwaddr, curr->hwaddr);
    }
    CONFD_SET_TAG_BOOL(&(tobj[i].tv[3]), arpe_permanent, curr->perm);
    CONFD_SET_TAG_BOOL(&(tobj[i].tv[4]), arpe_published, curr->pub);
    tobj[i].n = navals;
    tobj[i].next = (long)curr->next;
  }
  
  if (curr == NULL) {
    tobj[i++].tv = NULL;
  }
  
  confd_data_reply_next_object_tag_value_arrays(tctx, tobj, i, 0); /* Reply with an array of object arrays. I.e. the whole table / list */
  free(tv);
  free(tobj);
  return CONFD_OK;
}

int main(int argc, char *argv[])
{
...
    data.get_next_object = get_next_object_tag;
...
    if (confd_load_schemas((struct sockaddr*)&addr,
                           sizeof (struct sockaddr_in)) != CONFD_OK)
        confd_fatal("Failed to load schemas from confd\n");

    object = confd_cs_node_cd(NULL, "/arpe:arpentries/arpe");
    navals = confd_max_object_size(object);
...
}

Thanks . i addressed the issue, i use the same memory for different key, i found before calling confd_reply_XX, it just point to the memory.

I’m wondering that what will happen if the size of list in response is larger than the request one.i mean CONFD reply more lists(for example: 3 instances) , however we just ask 2 instances. it seems that CONFD does not know how many lists now is requesting

i want to CONFD return data of confd_tag_value_t type, which API should i should in client? as i know , maapi_get_objects just get confd_value_t

We are getting the size of one list entry / object / row here, not the number of list entries / instances / objects / rows

Try maapi_get_values() / cdb_get_values().

@cohult, Do you know how confd only trigger get_next_object for Netconf request and confd still trigger get_next/get_elem for CLI. We use the external data provider and need to support 8k onts. Since get_next_object back-end code will stuck some while, we don’t want cli stuck. So we want to know how to trigger get_next_object only for Netconf request.
Much appreciated.

I think it is recommended not to return huge amount of objects (8k) in get_next_object. You should limit it to cca 100 elements (make it perhaps configurable and try to find best value). Then CLI will be responsive as well. In several cases (e.g. TAB completion) ConfD can choose itself to call get_next instead of get_next_object

Thanks @mnovak. Yes, we don’t return all the instances in one swoop and are trying out to find the best cca value. But actually, there are no customers complaining cli performance issue, so we don’t want to impact on CLI. We only want to support get_next_object for netconf request. I see there are “bulk hint” mentioned in confd_user_guide_6.7.4.pdf, but there are no further detail description. So I want to confirm whether confd can only trigger get_next_object for netconf somehow.
Much appreciated.

HI @mnovak, I also have another block issue. Take a simple example: list ont
and list ont has nested-list vlan.
Since nested-list vlan can’t be returned to confd in the get_next_object for list ont, confd will trigger get_next_object for nested-list vlan. So there are large numbers of get_next_object for nested-list vlan, which is not efficient and consumes lots of time, right? So how to improve the loop of nested-list? By the way, “container children nodes” can be returned in get_next_object for list onts, and I use confd_data_reply_next_object_tag_value_arrays() and external data provider.
Much appreciated.

Hello,

unfortunately I do not know know how to force get_next_object just on NETCONF. Usually ConfD should choose best method. I suggest you implement get_next_object returning cca 100 records and test the implementation together with get_next (and maybe also find_next and find_next_object if needed). CLI performance should not be affected.

I thing for get_next_object same restriction applies as for get_object (see user guide - description of get_object):

However, although the tagged value array format can represent nested lists, these must not be passed via this function, since the get_object() callback only pertains to a single entry of one list. Nodes representing sub-lists must thus be omitted from the array, and ConfD will issue separate get_object() invocations to retrieve the data for those.

HI @mnovak, thanks for your confirm. Now, I hit another block issue. There are “when statement” and “must statement” under some nodes in list, such as “/config/product/machine!= 'E3-16F”. Since there are substantial amounts of list instances, confd trigger lots of get-elem() request to validate “when/must statement”. I want to know whether confd can cache the “when/must statement” result and only trigger the first time for the same xpath.

Please refer to the below fragment:

DEBUG> 25-Aug-2019::20:15:12.961 Jessica-is-using confd[6524]: devel-c get_elem request for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.963 Jessica-is-using confd[6524]: devel-c get_elem succeeded for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.967 Jessica-is-using confd[6524]: devel-c get_elem request for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.969 Jessica-is-using confd[6524]: devel-c get_elem succeeded for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.971 Jessica-is-using confd[6524]: devel-c get_elem request for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.974 Jessica-is-using confd[6524]: devel-c get_elem succeeded for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.976 Jessica-is-using confd[6524]: devel-c get_elem request for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.979 Jessica-is-using confd[6524]: devel-c get_elem succeeded for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.983 Jessica-is-using confd[6524]: devel-c get_elem request for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.986 Jessica-is-using confd[6524]: devel-c get_elem succeeded for callpoint config_cp path /exa:config/product/machine
25-Aug-2019::20:15:12.988 Jessica-is-using confd[6524]: devel-c get_elem request for callpoint config_cp path /exa:config/product/machine

You may try to use ConfD cache for operational data. See ConfD user guide for confd.conf, search for /confdConfig/opcache.

Thanks @mnovak. I tried out this /confdConfig/opcache in confd.conf by reference to ConfD user guide . But it doesn’t work for my case. My case is running configuration data and get-elem() requests are triggered by “when” validation.