前情提要

上一篇:C++实现神经网络框架(一):从表达式树到自动微分框架的思路解读与代码实现

#define _DEBUG_
#include"Matrix.h"
#include"AutoGraph.h"
#include<iostream>
using namespace std;
int main()
{AutoGraph<Matrix<int>>a={{3,3},{4,4}};AutoGraph<Matrix<int>>b={{1,2},{3,2}};AutoGraph<Matrix<int>>c={{1,2},{3,2}};AutoGraph<Matrix<int>>d={{1,2},{1,1}};AutoGraph<Matrix<int>>ans=(a*b*c-b*c)*(d+a);cout<<"---延迟计算,根据表达式自动构建计算图,但不进行计算---\n"; cout<<"---打印表达式---\n";ans.plot_tree();cout<<"--------开始计算--------\n"; cout<<ans.one_shot_apply_data();
} 

目前我们仅实现了四则运算
但是实际上我们还有很多运算需要添加
比如矩阵转置x=x.transpose(),矩阵求对数=x=x.log()
或者matmul(a,b)这种二元函数
也或者RandomNormal(stddev(5),mean(6))这样的数据生成节点
所以我们要继续对代码进行改进

node_type

之前的命名不太好,改了一下

namespace node_type
{constexpr int none=0;constexpr int data=1; //数据 constexpr int cond=2; //分支控制 ans=cond(boolen_expression,expression a,expression b)constexpr int binary_operator=3; //ans=a*bconstexpr int univariate_function=4; //一元函数 ans=log(a)constexpr int binary_function=5; //二元函数 ans=matlum(a,b) 
};

Node

在Node类中添加必要的构造函数和function对象
部分更新的代码如下

template<typename Dtype>
class Node
{ 
public:...Node(Node*lnode,Node*rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name="opt",int type=node_type::binary_operator);  //构造操作节点,该结点作为lnode rnode两个结点的父节点 Node(Node*node,std::function<Dtype&(Dtype&)>binary_function,std::string name="opt",int type=node_type::univariate_function); void plot_expression()const;                     //中序遍历树(检查正确性) ...
private:...                            std::function<Dtype&(Dtype&,Dtype&)>binary_function;  //接受一个二元函数,this->data=&this->binary_function(this->lchild-Ldata,this>rchild->data) std::function<Dtype&(Dtype&)>univariate_function;  //接受一个无参函数 this->data=&this->univariate_function(this->lchild->data)...	
};
template<typename Dtype>
Node<Dtype>::Node(Node*node,std::function<Dtype&(Dtype&)>univariate_function,std::string name,int type)
{this->visit=0;this->data=nullptr;this->lchild=node;this->rchild=nullptr;this->univariate_function=univariate_function;this->name=name;this->type=type;
}

AutoGraph

然后再对应的AutoGraph中也添加相应的构造函数

template<typename Dtype>
class AutoGraph
{
public:AutoGraph(const AutoGraph<Dtype>&lnode,AutoGraph<Dtype>&rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name="binary_operator",int type=node_type::binary_operator);AutoGraph(AutoGraph<Dtype>&lnode,AutoGraph<Dtype>&rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name="binary_function",int type=node_type::binary_function);AutoGraph(AutoGraph<Dtype>&node,std::function<Dtype&(Dtype&)>univariate_function,std::string name="univariate_function",int type=node_type::binary_function);
private:Node<Dtype>*root;
};
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(AutoGraph<Dtype>&child,std::function<Dtype&(Dtype&)>univariate_function,std::string name,int type)
{this->root=new Node<Dtype>(child.root,univariate_function,name,type);
}

macro

添加一些宏函数来方便写代码

//AutoGraph.h
#define LambdaBinaryOps(type,a,b,context) [](type a,type b)->type{return context;}
#define LambdaSingleOps(type,a,context) [](type a)->type{return context;}
//autograph_operator_register.h
#define autograph_register_binary_method(Cls,method)\
template<typename Dtype>\
AutoGraph<Cls<Dtype>>&method(AutoGraph<Cls<Dtype>>&a,AutoGraph<Cls<Dtype>>&b)\
{\return *new AutoGraph<Cls<Dtype>>(a,b,LambdaBinaryOps(Cls<Dtype>&,x,y,ensure_reference(x.method(y))),#method,node_type::binary_function);\
}
#define autograph_register_single_method(Cls,method)\
template<typename Dtype>\
AutoGraph<Cls<Dtype>>&method(AutoGraph<Cls<Dtype>>&a)\
{\return *new AutoGraph<Cls<Dtype>>(a,LambdaSingleOps(Cls<Dtype>&,x,ensure_reference(x.method())),#method,node_type::univariate_function);\
}

然后就是添加一些操作进去

#ifndef _AUTOGRAPH_OPERATOR_REGISTER_H_
#define _AUTOGRAPH_OPERATOR_RESIGTER_H_ 
autograph_register_binary_method(Matrix,matmul);
autograph_register_binary_method(Matrix,greater);
autograph_register_binary_method(Matrix,less);
autograph_register_single_method(Matrix,transpose);
autograph_register_single_method(Matrix,abs);
autograph_register_single_method(Matrix,square);
autograph_register_single_method(Matrix,exp);
autograph_register_single_method(Matrix,log);
#endif 

测试代码

#define _DEBUG_MEMORY_
#include"Matrix.h"
#include"AutoGraph.h"
#include"autograph_operator_register.h"
#include<iostream>
using std::cout;
using std::endl;
int main()
{AutoGraph<Matrix<double>>a={{1,2,3},{4,5,6}};AutoGraph<Matrix<double>>b={{4,5,6},{7,8,1}};AutoGraph<Matrix<double>>c={{1,2},{3,4},{5,6}};AutoGraph<Matrix<double>>d={{1,2},{4,3}};AutoGraph<Matrix<double>>ans=(matmul(a*b,c))-d;ans=log(d+square(ans-d))-d;ans=abs(log(log(log((ans)))));ans=less(ans,d);cout<<ans.one_shot_apply_data();
//	ans.clear_all();
}

#define _DEBUG_
#include"Matrix.h"
#include"AutoGraph.h"
#include"autograph_operator_register.h"
#include<iostream>
using std::cout;
using std::endl;
int main()
{AutoGraph<Matrix<double>>a={{1,2,3},{4,5,6}};AutoGraph<Matrix<double>>b={{4,5,6},{7,8,1}};AutoGraph<Matrix<double>>c={{1,2},{3,4},{5,6}};AutoGraph<Matrix<double>>d={{1,2},{4,3}};AutoGraph<Matrix<double>>ans=(matmul(a*b,c))-d;ans=log(d+square(ans-d))-d;ans=abs(log(log(log((ans)))));ans=less(ans,d);cout<<ans.apply_data();ans.clear_all();
}

#include"Matrix.h"
#include"AutoGraph.h"
#include"autograph_operator_register.h"
#include<iostream>
using std::cout;
using std::endl;
int main()
{AutoGraph<Matrix<double>>a={{1,2,3},{4,5,6}};AutoGraph<Matrix<double>>b={{4,5,6},{7,8,1}};AutoGraph<Matrix<double>>c={{1,2},{3,4},{5,6}};AutoGraph<Matrix<double>>d={{1,2},{4,3}};AutoGraph<Matrix<double>>ans=(matmul(a*b,c))-d;ans=log(d+square(ans-d))-d;ans=abs(log(log(log((ans)))));ans=less(ans,d);ans.plot_expression(); 
}

程序结果

目前为止内存和计算上正常,没有出现内存泄漏的问题
不过该程序目前有点小BUG,在某种特殊情况下下指针的删除存在问题
就是拓扑排序上出问题了
暂时为解决emmm
先就这样了
解决这个之后
下一个任务就是Layer的前向传播和反向传播

部分主要代码

AutoGraph.h

#ifndef _AUTOGRAPH_H
#define _AUTOGRAPH_H 
#include<iostream>
#include<cstring>
#include<functional>
#include<cmath>
#include"Matrix.h"
#define LambdaBinaryOps(type,a,b,context) [](type a,type b)->type{return context;}
#define LambdaSingleOps(type,a,context) [](type a)->type{return context;}
namespace node_type
{constexpr int none=0;constexpr int data=1; //数据 constexpr int cond=2; //分支控制 ans=cond(boolen_expression,expression a,expression b)constexpr int binary_operator=3; //ans=a*bconstexpr int univariate_function=4; //一元函数 ans=log(a)constexpr int binary_function=5; //二元函数 ans=matlum(a,b) 
};
template<typename T> 
typename std::enable_if<std::is_fundamental<T>::value,T>::type& ensure_reference(T a)
{return *new T(a);
} //如果是基本类型,按值传参返回一个引用 
template<typename T>
typename std::enable_if<!std::is_fundamental<T>::value,T>::type& ensure_reference(T&a)
{return a;
} //如果是一个class,传入引用,返回自己
template<bool cond,typename T>
struct enable_reference
{typedef T type;
};
template<typename T>                          //enable_reference<is_fundamental<T>::value,T>::type 
struct enable_reference<false,T>              //如果是基本类型,按值传参,否则传递引用  
{                                             //class cls{}; typedef T& type;                          //cout<<is_reference<enable_reference<is_fundamental<cls>::value,cls>::type>::value; //true 
};                                            //cout<<is_reference<enable_reference<is_fundamental<int>::value,int>::type>::value; //false 
template<typename Dtype>
class Node
{ 
public:Node();Node(Dtype&data,std::string name="data");  //构造数据节点 Node(Node*lnode,Node*rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name="opt",int type=node_type::binary_operator);  //构造操作节点,该结点作为lnode rnode两个结点的父节点 Node(Node*node,std::function<Dtype&(Dtype&)>binary_function,std::string name="opt",int type=node_type::binary_operator); ~Node();Node(const Node&node);Dtype&apply_data();                        //计算该节点的值 Dtype&one_shot_apply_data();void plot_expression()const;                     //中序遍历树(检查正确性) void clear();                              //清空中间过程的值 void clear_all();                          //清空所有值 
private:Dtype*data;                                std::function<Dtype&(Dtype&,Dtype&)>binary_function;  //接受一个二元函数,this->data=&this->binary_function(this->lchild-Ldata,this>rchild->data) std::function<Dtype&(Dtype&)>univariate_function;  //接受一个无参函数 this->data=&this->univariate_function(this->lchild->data)Node*lchild,*rchild;int type,visit;std::string name;                          //该节点的名称 
};
template<typename Dtype>
Node<Dtype>::Node(const Node<Dtype>&node)
{this->visit=node.visit;this->data=new Dtype;this->lchild=new Node<Dtype>;this->rchild=new Node<Dtype>;this->binary_function=node.binary_function;this->name=node.name;this->type=node.type;*this->data=*node.data;*this->lchild=*node.lchild;*this->rchild=*node.rchild;
}
template<typename Dtype>
Node<Dtype>::Node()
{this->visit=0;this->data=nullptr;this->lchild=nullptr;this->rchild=nullptr;this->name="";this->type=node_type::none; 
}
template<typename Dtype>
Node<Dtype>::Node(Dtype&data,std::string name)
{ this->visit=0;this->data=&data;this->lchild=nullptr;this->rchild=nullptr;this->name=name;this->type=node_type::data;
}
template<typename Dtype>
Node<Dtype>::Node(Node*lnode,Node*rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name,int type)
{this->visit=0;this->data=nullptr;this->lchild=lnode;this->rchild=rnode;this->binary_function=binary_function;this->name=name;this->type=type;
}
template<typename Dtype>
Node<Dtype>::Node(Node*node,std::function<Dtype&(Dtype&)>univariate_function,std::string name,int type)
{this->visit=0;this->data=nullptr;this->lchild=node;this->rchild=nullptr;this->univariate_function=univariate_function;this->name=name;this->type=type;
}
template<typename Dtype>
Node<Dtype>::~Node() //递归删掉整个子树 
{if(this->data)delete data;if(this->lchild)delete this->lchild;if(this->rchild)delete this->rchild;this->lchild=nullptr;this->rchild=nullptr; this->data=nullptr;
}
template<typename Dtype>
void Node<Dtype>::clear() //递归删掉整个子树被计算出来的数据
{if(this->type!=node_type::data){delete this->data;this->data=nullptr;}if(this->lchild)this->lchild->clear();if(this->rchild)this->rchild->clear();
#ifdef _DEBUG_ std::cout<<"name:"<<this->name<<" type: "<<this->type<<" data: "<<this->data<<std::endl;
#endif
}
#ifdef _DEBUG_
template<typename Dtype>
void Node<Dtype>::clear_all() //递归删掉所有数据,但是树结构保留了,用来测试的 
{if(this->data){delete this->data;this->data=nullptr;}if(this->lchild)this->lchild->clear_all();if(this->rchild)this->rchild->clear_all();
//	std::cout<<"name:"<<this->name<<" type: "<<this->type<<" data: "<<this->data<<std::endl;
}
#endif
template<typename Dtype>
void Node<Dtype>::plot_expression()const
{if(this->type==node_type::binary_function||this->type==node_type::univariate_function)std::cout<<this->name;if(this->type==node_type::binary_function||this->type==node_type::univariate_function||this->type==node_type::binary_operator)std::cout<<'(';if(this->lchild)this->lchild->plot_expression();if(this->type==node_type::data)std::cout<<(*this->data); else{if(this->type==node_type::binary_operator)std::cout<<" "<<this->name<<" ";if(this->type==node_type::binary_function)std::cout<<",";               }if(this->rchild)this->rchild->plot_expression();if(this->type==node_type::binary_operator||this->type==node_type::univariate_function||this->type==node_type::binary_function)std::cout<<')';
}
template<typename Dtype>
Dtype&Node<Dtype>::apply_data()
{if(!this->data){if(this->type==node_type::binary_function||this->type==node_type::binary_operator)this->data=&this->binary_function(this->lchild->apply_data(),this->rchild->apply_data());else if(this->type==node_type::univariate_function)this->data=&this->univariate_function(this->lchild->apply_data());			}return *this->data;
}
template<typename Dtype>
Dtype&Node<Dtype>::one_shot_apply_data() //根节点数据没有被删除,因此析构函数少调用了一次.
{if(!this->data){if(this->type==node_type::binary_function||this->type==node_type::binary_operator){this->lchild->visit++;this->rchild->visit++;this->data=&this->binary_function(this->lchild->one_shot_apply_data(),this->rchild->one_shot_apply_data());this->lchild->visit--;this->rchild->visit--;}else if(this->type==node_type::univariate_function){this->lchild->visit++;this->data=&this->univariate_function(this->lchild->one_shot_apply_data());this->lchild->visit--;}}if(this->lchild&&this->lchild->data&&this->lchild->visit==0){delete this->lchild->data;this->lchild->data=NULL;}if(this->rchild&&this->rchild->data&&this->rchild->visit==0){delete this->rchild->data;this->rchild->data=NULL;}return*this->data;
}
template<typename Dtype>
class AutoGraph
{
public:AutoGraph(typename Dtype::ElemType constant);AutoGraph(typename Dtype::InitializerType a);AutoGraph(Dtype data);AutoGraph(Node<Dtype>&root);AutoGraph(const AutoGraph<Dtype>&lnode,AutoGraph<Dtype>&rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name="binary_operator",int type=node_type::binary_operator);AutoGraph(AutoGraph<Dtype>&lnode,AutoGraph<Dtype>&rnode,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name="binary_function",int type=node_type::binary_function);AutoGraph(AutoGraph<Dtype>&node,std::function<Dtype&(Dtype&)>univariate_function,std::string name="univariate_function",int type=node_type::binary_function);virtual ~AutoGraph(){}AutoGraph&operator+(AutoGraph&)const;AutoGraph&operator*(AutoGraph&)const;AutoGraph&operator-(AutoGraph&)const;AutoGraph&operator/(AutoGraph&)const;void plot_expression()const;   //打印结点 void clear();            //清空数据 void clear_all();        //清空所有数据(DEBUG) Dtype&apply_data();      //计算该节点的值 Dtype&one_shot_apply_data();      //计算该节点的值 Node<Dtype>*get_root()const; //得到树根 
private:Node<Dtype>*root;
};
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(const AutoGraph<Dtype>&lchild,AutoGraph<Dtype>&rchild,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name,int type)
{this->root=new Node<Dtype>(lchild.root,rchild.root,binary_function,name,type);
}
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(AutoGraph<Dtype>&lchild,AutoGraph<Dtype>&rchild,std::function<Dtype&(Dtype&,Dtype&)>binary_function,std::string name,int type)
{this->root=new Node<Dtype>(lchild.root,rchild.root,binary_function,name,type);
}
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(AutoGraph<Dtype>&child,std::function<Dtype&(Dtype&)>univariate_function,std::string name,int type)
{this->root=new Node<Dtype>(child.root,univariate_function,name,type);
}
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(Node<Dtype>&root)
{this->root=&root;
}
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(typename Dtype::ElemType constant)
{this->root=new Node<Dtype>(*new Dtype(constant));
}
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(typename Dtype::InitializerType data) //InitializerType 是初始化Matrix用的类型,这里给AutoGraph构造函数偏特化一个版本出来 
{this->root=new Node<Dtype>(*new Dtype(data));
}
template<typename Dtype>
AutoGraph<Dtype>::AutoGraph(Dtype data)
{this->root=new Node<Dtype>(*new Dtype(data));
}
template<typename Dtype>
AutoGraph<Dtype>&AutoGraph<Dtype>::operator+(AutoGraph<Dtype>&b)const //任何一种二元运算符操作都会产生一个新结点作为根结点 
{return *new AutoGraph<Dtype>(*this,b,LambdaBinaryOps(Dtype&,a,b,ensure_reference(a+b)),"+");
}
template<typename Dtype>
AutoGraph<Dtype>&AutoGraph<Dtype>::operator-(AutoGraph<Dtype>&b)const //任何一种二元运算符操作都会产生一个新结点作为根结点 
{return *new AutoGraph<Dtype>(*this,b,LambdaBinaryOps(Dtype&,a,b,ensure_reference(a-b)),"-");
}
template<typename Dtype>
AutoGraph<Dtype>&AutoGraph<Dtype>::operator*(AutoGraph<Dtype>&b)const //表达式树的构建就是二叉树不断合并的过程 
{return *new AutoGraph<Dtype>(*this,b,LambdaBinaryOps(Dtype&,a,b,ensure_reference(a*b)),"·");
}
template<typename Dtype>
AutoGraph<Dtype>&AutoGraph<Dtype>::operator/(AutoGraph<Dtype>&b)const 
{return *new AutoGraph<Dtype>(*this,b,LambdaBinaryOps(Dtype&,a,b,ensure_reference(a/b)),"/");
}
template<typename Dtype>
Dtype&AutoGraph<Dtype>::apply_data()
{return root->apply_data();
}
template<typename Dtype>
Dtype&AutoGraph<Dtype>::one_shot_apply_data()
{return root->one_shot_apply_data();
}
template<typename Dtype>
void AutoGraph<Dtype>::plot_expression()const
{this->root->plot_expression();
}
template<typename Dtype>
void AutoGraph<Dtype>::clear()
{this->root->clear();
}
template<typename Dtype>
void AutoGraph<Dtype>::clear_all()
{this->root->clear_all();
}
template<typename Dtype>
Node<Dtype>*AutoGraph<Dtype>::get_root()const
{	return this->root;
} 
#endif

autograph_operator_register.h

#ifndef _AUTOGRAPH_OPERATOR_REGISTER_H_
#define _AUTOGRAPH_OPERATOR_RESIGTER_H_ 
#include"Matrix.h"
#include"AutoGraph.h"
#define autograph_register_binary_method(Cls,method)\
template<typename Dtype>\
AutoGraph<Cls<Dtype>>&method(AutoGraph<Cls<Dtype>>&a,AutoGraph<Cls<Dtype>>&b)\
{\return *new AutoGraph<Cls<Dtype>>(a,b,LambdaBinaryOps(Cls<Dtype>&,x,y,ensure_reference(x.method(y))),#method,node_type::binary_function);\
}
#define autograph_register_single_method(Cls,method)\
template<typename Dtype>\
AutoGraph<Cls<Dtype>>&method(AutoGraph<Cls<Dtype>>&a)\
{\return *new AutoGraph<Cls<Dtype>>(a,LambdaSingleOps(Cls<Dtype>&,x,ensure_reference(x.method())),#method,node_type::univariate_function);\
}
autograph_register_binary_method(Matrix,matmul);
autograph_register_binary_method(Matrix,greater);
autograph_register_binary_method(Matrix,less);
autograph_register_single_method(Matrix,transpose);
autograph_register_single_method(Matrix,abs);
autograph_register_single_method(Matrix,square);
autograph_register_single_method(Matrix,exp);
autograph_register_single_method(Matrix,log);
autograph_register_single_method(Matrix,pow);
#endif