ConfD User Community

How to add the support of the find_next_object() callback to the 5-c_stats example?


#1

Implementing the find_next_object() callback can be a significant optimization for large lists in cases where a northbound agent, e.g. over NETCONF, requests a specific part of the list. Similar to the get_next_object() that we saw in action in this post, the find_next_object() callback combines find_next() and get_object() into a single callback and optimizes cases where ConfD wants to start a list traversal at some other point than at the first entry of the list and return an array of list instances in one single swoop.
As with get_next_object(), find_next_object() is mainly useful for lists with a large number of entries.

The changes needed to add the find_next_object() callback support to the 5-c_stats example will be described in this posting. If you have an application that benefits from implementing find_next_object(), it is likely that you will benefit from implementing the get_next_object() callback too, so this post assumes that you already made the changes from the get_next_object() post to add that callback to your 5-c_stats example. All the changes needed to implement the find_next_object() callback will be made to the arpstat.c file of the example.
In the main() function, you will add the registration of your find_next_object() callback as follows:

data.find_next_object = find_next_object;

The implementation of the find_next_object() callback for the 5-c_stats example use-case:

static int find_next_object(struct confd_trans_ctx *tctx,
                            confd_hkeypath_t *kp,
                            enum confd_find_next_type type,
                            confd_value_t *keys, int nkeys)
{

  int pos = -1;
  int ipos;

  confd_value_t *v;
  struct confd_next_object *obj;
  int i;

  struct arpdata *dp = tctx->t_opaque;
  int n_list_entries;
  struct aentry *ae;
  struct in_addr ip;
  char *iface;

  if (need_arp(dp)) {
    if (run_arp(dp) == CONFD_ERR)
      return CONFD_ERR;
  }

  n_list_entries = dp->n_ae;
  ae = dp->arp_entries;

  switch (nkeys) {
  case 0:
    if(n_list_entries > 0) {
      pos = 0; /* No keys provided => the first entry will always be "after" */
    }
    break;
  case 1:
    /* one key provided => find the first entry "after"
       - since there can never be a "same" entry, 'type' is not relevant,
       and an entry with same first key is also "after" */
    ip = CONFD_GET_IPV4(&keys[0]);
    ipos = 0;
    for (; ae != NULL; ae = ae->next) {
      if (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) >= 0) {
        pos = ipos;
        break;
      }
      ipos++;
    }
    break;
  case 2:
    /* both keys provided => find first entry "after" or "same",
       depending on 'type' */
    ip = CONFD_GET_IPV4(&keys[0]);
    iface = (char*)CONFD_GET_BUFPTR(&keys[1]);

    switch (type) {
    case CONFD_FIND_NEXT:
      /* entry must be "after" */
      ipos = 0;
      for (; ae != NULL; ae = ae->next) {
      if (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) > 0 ||
            (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) == 0 &&
             strcmp(iface, ae->iface) > 0)) {
        pos = ipos;
        break;
      }
    ipos++;
    }
    break;
    case CONFD_FIND_SAME_OR_NEXT:
      /* entry must be "same" or "after" */
      ipos = 0;
      for (; ae != NULL; ae = ae->next) {
        if (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) > 0 ||
            (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) == 0 &&
             strcmp(iface, ae->iface) >= 0)) {
          pos = ipos;
          break;
        }
        ipos++;
      }
      break;
    }
    break;
  default:
    confd_trans_seterr(tctx, "invalid number of keys: %d", nkeys);
    return CONFD_ERR;
  }

  v = (confd_value_t *) malloc(sizeof(confd_value_t) * n_list_entries * N_LEAFS);
  obj = malloc(sizeof(struct confd_next_object) * (n_list_entries + 2));

  if (pos != -1) {
    for (i = 0; pos + i < n_list_entries; i++) {
      obj[i].v = &v[i * N_LEAFS];

      CONFD_SET_IPV4(&(obj[i].v[0]), ae->ip4);
      CONFD_SET_STR(&(obj[i].v[1]), ae->iface);
      if (ae->hwaddr == NULL) {
        CONFD_SET_NOEXISTS(&(obj[i].v[2]));
      } else {
        CONFD_SET_STR(&(obj[i].v[2]), ae->hwaddr);
      }
      CONFD_SET_BOOL(&(obj[i].v[3]), ae->perm);
      CONFD_SET_BOOL(&(obj[i].v[4]), ae->pub);

      obj[i].n = N_LEAFS;
      obj[i].next = -1;
      ae = ae->next;
    }
    if (pos + i == n_list_entries)
      obj[i++].v = NULL;
    confd_data_reply_next_object_arrays(tctx, obj, i, 0);
  } else {
    confd_data_reply_next_object_array(tctx, NULL, -1, -1);
  }
  free(v);
  free(obj);
  return CONFD_OK;
}

After the above changes are made to the arpstat.c file, you will then be able to rebuild your project and see the find_next_object() callback in action through the TRACE statements of the arpstat program.
Say we have an ARP table with only eight entries to keep it simple for this example:

$ arp -a
apps-net0-5.test.com (172.16.171.6) at dd:aa:aa:aa:aa:aa [ether] PERM on eth1
apps-net0-2.test.com (172.16.171.3) at aa:aa:aa:aa:aa:aa [ether] PERM on eth0
apps-net5-4.test.com (172.16.171.254) at 00:50:56:f3:13:d2 [ether] on eth2
apps-net0-4.test.com (172.16.171.5) at cc:aa:aa:aa:aa:aa [ether] PERM on eth1
apps-net0-6.test.com (172.16.171.7) at ee:aa:aa:aa:aa:aa [ether] PERM on eth1
apps-net0-3.test.com (172.16.171.4) at ff:aa:aa:aa:aa:aa [ether] PERM on eth1
apps-net0-3.test.com (172.16.171.4) at bb:aa:aa:aa:aa:aa [ether] PERM on eth0
apps-net0-1.test.com (172.16.171.2) at 00:50:56:f7:9d:e2 [ether] on eth2

We <GET> a specific part of the list, i.e. all entries with IP address 172.16.171.4, over NETCONF using the netconf-console tool that comes with ConfD:

$ netconf-console --get -x '/arpentries/arpe[ip="172.16.171.4"][ifname]/hwaddr'
<?xml version="1.0" encoding="UTF-8"?>
<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="1">
  <data>
    <arpentries xmlns="http://tail-f.com/ns/example/arpe">
      <arpe>
        <ip>172.16.171.4</ip>
        <ifname>eth0</ifname>
        <hwaddr>bb:aa:aa:aa:aa:aa</hwaddr>
      </arpe>
      <arpe>
        <ip>172.16.171.4</ip>
        <ifname>eth1</ifname>
        <hwaddr>ff:aa:aa:aa:aa:aa</hwaddr>
      </arpe>
    </arpentries>
  </data>
</rpc-reply>

As we can see from the TRACE statements of the arpstat program, all list instances with IP address 172.16.171.4 was fetched in one single swoop by ConfD from the application using the find_next_object() callback:

TRACE New user session: 17 for user:admin ctx:netconf --> CONFD_OK
TRACE CALL trans init(thandle=12,mode="r",db=running) --> CONFD_OK
TRACE CALL data find_next_object(thandle=12, /arpentries/arpe, same_or_next, {172.16.171.4}) --> CONFD_OK
TRACE CALL trans finish(thandle=12) --> CONFD_OK
TRACE Close user sess 17
 --> CONFD_OK

Multiple threads
#2

An updated variant that support returning the next object instead of “-1” and not hard coding the number of objects in each array.
More importantly this variant chunk a max number of objects / list entries / rows for improved performace.

...
static int navals;
static int max_nobjs = 100; // Chunk size hardcoded here to simplify
...
static int find_next_object(struct confd_trans_ctx *tctx,
                            confd_hkeypath_t *kp,
                            enum confd_find_next_type type,
                            confd_value_t *keys, int nkeys)
{
  
  int pos = -1;
  int ipos;

  confd_value_t *v;
  struct confd_next_object *obj;
  int i;
  struct arpdata *dp = tctx->t_opaque;
  int n_list_entries, nobjs;
  struct aentry *ae;
  struct in_addr ip;
  char *iface;
  
  if (need_arp(dp)) {
    if (run_arp(dp) == CONFD_ERR)
      return CONFD_ERR;
  }

  n_list_entries = dp->n_ae;
  ae = dp->arp_entries;
  
  switch (nkeys) {
  case 0:
    if(n_list_entries > 0) {
      pos = 0; /* No keys provided => the first entry will always be "after" */
    }
    break;
  case 1:
    /* one key provided => find the first entry "after"
       - since there can never be a "same" entry, 'type' is not relevant,
       and an entry with same first key is also "after" */
    ip = CONFD_GET_IPV4(&keys[0]);
    ipos = 0;
    for (; ae != NULL; ae = ae->next) {
      if (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) >= 0) {
	pos = ipos;
	break;
      }
      ipos++;
    }
    break;
  case 2:
    /* both keys provided => find first entry "after" or "same",
       depending on 'type' */
    ip = CONFD_GET_IPV4(&keys[0]);
    iface = (char*)CONFD_GET_BUFPTR(&keys[1]);

    switch (type) {
    case CONFD_FIND_NEXT:
      /* entry must be "after" */
      ipos = 0; 
      for (; ae != NULL; ae = ae->next) {
	if (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) > 0 ||
            (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) == 0 &&
             strcmp(iface, ae->iface) > 0)) {
	  pos = ipos;
	  break;
	}
	ipos++;
      }
      break;
    case CONFD_FIND_SAME_OR_NEXT:
      /* entry must be "same" or "after" */
      ipos = 0;
      for (; ae != NULL; ae = ae->next) {
	if (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) > 0 ||
            (memcmp(&ae->ip4, &ip, sizeof(struct in_addr)) == 0 &&
             strcmp(iface, ae->iface) >= 0)) {
	  pos = ipos;
	  break;
	}
	ipos++;
      }
      break;
    }
    break;
  default:
    confd_trans_seterr(tctx, "invalid number of keys: %d", nkeys);
    return CONFD_ERR;
  }

  if (n_list_entries - pos > max_nobjs) {
    nobjs = max_nobjs + pos;
  } else {
    nobjs = n_list_entries;
  }
  
  v = (confd_value_t *) malloc(sizeof(confd_value_t) * nobjs * navals);
  obj = malloc(sizeof(struct confd_next_object) * (nobjs + 2));

  if (pos != -1) {
    for (i = 0; pos + i < nobjs; i++) {
      obj[i].v = &v[i * navals];

      CONFD_SET_IPV4(&(obj[i].v[0]), ae->ip4);
      CONFD_SET_STR(&(obj[i].v[1]), ae->iface);
      if (ae->hwaddr == NULL) {
	CONFD_SET_NOEXISTS(&(obj[i].v[2]));
      } else {
	CONFD_SET_STR(&(obj[i].v[2]), ae->hwaddr);
      }
      CONFD_SET_BOOL(&(obj[i].v[3]), ae->perm);
      CONFD_SET_BOOL(&(obj[i].v[4]), ae->pub);

      obj[i].n = navals;
      obj[i].next = (long)ae->next;//-1;
      ae = ae->next;
    }
    if (pos + i == n_list_entries)
      obj[i++].v = NULL;
    confd_data_reply_next_object_arrays(tctx, obj, i, 0);
  } else {
    confd_data_reply_next_object_array(tctx, NULL, -1, -1);
  }
  free(v);
  free(obj);
  return CONFD_OK;
}
...
int main(int argc, char *argv[])
{
    struct confd_cs_node *object;
...
    object = confd_cs_node_cd(NULL, "/arpe:arpentries/arpe");
    navals = confd_max_object_size(object);
...